port release automations from arrow-rs (#780)

* port release automations from arrow-rs

* remove cherry-pick-pr

.github_changelog_generator

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# point to the old changelog in apache/arrow
+front-matter=For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)\n
+# some issues are just documentation
+add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]}}
+# uncomment to not show PRs. TBD if we shown them or not.
+#pull-requests=false
+# so that the component is shown associated with the issue
+issue-line-labels=ballista,datafusion,python
+exclude-labels=development-process,invalid
+breaking_labels=api-change

dev/release/create-tarball.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/create-tarball.sh
+
+# This script creates a signed tarball in
+# dev/dist/apache-arrow-datafusion-<version>-<sha>.tar.gz and uploads it to
+# the "dev" area of the dist.apache.arrow repository and prepares an
+# email for sending to the dev@arrow.apache.org list for a formal
+# vote.
+#
+# See release/README.md for full release instructions
+#
+# Requirements:
+#
+# 1. gpg setup for signing and have uploaded your public
+# signature to https://pgp.mit.edu/
+#
+# 2. Logged into the apache svn server with the appropriate
+# credentials
+#
+#
+# Based in part on 02-source.sh from apache/arrow
+#
+
+set -e
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <tag> <rc>"
+    echo "ex. $0 4.1.0 2"
+  exit
+fi
+
+tag=$1
+rc=$2
+
+release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})
+
+release=apache-arrow-datafusion-${tag}
+distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc}
+tarname=${release}.tar.gz
+tarball=${distdir}/${tarname}
+url="https://dist.apache.org/repos/dist/dev/arrow/${release}-rc${rc}"
+
+echo "Attempting to create ${tarball} from tag ${tag}"
+
+
+if [ -z "$release_hash" ]; then
+    echo "Cannot continue: unknown git tag: $tag"
+fi
+
+echo "Draft email for dev@arrow.apache.org mailing list"
+echo ""
+echo "---------------------------------------------------------"
+cat <<MAIL
+To: dev@arrow.apache.org
+Subject: [VOTE][RUST][Datafusion] Release Apache Arrow Datafusion ${tag} RC${rc}
+Hi,
+I would like to propose a release of Apache Arrow Datafusion Implementation, version ${tag}.
+This release candidate is based on commit: ${release_hash} [1]
+The proposed release tarball and signatures are hosted at [2].
+The changelog is located at [3].
+Please download, verify checksums and signatures, run the unit tests,
+and vote on the release.
+The vote will be open for at least 72 hours.
+[ ] +1 Release this as Apache Arrow Datafusion ${version}
+[ ] +0
+[ ] -1 Do not release this as Apache Arrow Datafusion ${version} because...
+[1]: https://github.com/apache/arrow-datafusion/tree/${release_hash}
+[2]: ${url}
+[3]: https://github.com/apache/arrow-datafusion/blob/${release_hash}/CHANGELOG.md
+MAIL
+echo "---------------------------------------------------------"
+
+
+# create <tarball> containing the files in git at $release_hash
+# the files in the tarball are prefixed with {tag} (e.g. 4.0.1)
+mkdir -p ${distdir}
+(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball})
+
+echo "Running rat license checker on ${tarball}"
+${SOURCE_DIR}/run-rat.sh ${tarball}
+
+echo "Signing tarball and creating checksums"
+gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
+# create signing with relative path of tarball
+# so that they can be verified with a command such as
+#  shasum --check apache-arrow-datafusion-4.1.0-rc2.tar.gz.sha512
+(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256
+(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512
+
+echo "Uploading to apache dist/dev to ${url}"
+svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow ${SOURCE_TOP_DIR}/dev/dist
+svn add ${distdir}
+svn ci -m "Apache Arrow Datafusion ${tag} ${rc}" ${distdir}
+

dev/release/release-tarball.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/release-tarball.sh
+
+# This script copies a tarball from the "dev" area of the
+# dist.apache.arrow repository to the "release" area
+#
+# This script should only be run after the release has been approved
+# by the arrow PMC committee.
+#
+# See release/README.md for full release instructions
+#
+# Based in part on post-01-upload.sh from apache/arrow
+
+
+set -e
+set -u
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <rc-num>"
+  echo "ex. $0 4.1.0 2"
+  exit
+fi
+
+version=$1
+rc=$2
+
+tmp_dir=tmp-apache-arrow-datafusion-dist
+
+echo "Recreate temporary directory: ${tmp_dir}"
+rm -rf ${tmp_dir}
+mkdir -p ${tmp_dir}
+
+echo "Clone dev dist repository"
+svn \
+  co \
+  https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-${version}-rc${rc} \
+  ${tmp_dir}/dev
+
+echo "Clone release dist repository"
+svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release
+
+echo "Copy ${version}-rc${rc} to release working copy"
+release_version=arrow-datafusion-${version}
+mkdir -p ${tmp_dir}/release/${release_version}
+cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
+svn add ${tmp_dir}/release/${release_version}
+
+echo "Commit release"
+svn ci -m "Apache Arrow Datafusion ${version}" ${tmp_dir}/release
+
+echo "Clean up"
+rm -rf ${tmp_dir}
+
+echo "Success! The release is available here:"
+echo "  https://dist.apache.org/repos/dist/release/arrow/${release_version}"

dev/release/update_change_log.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/update_change_log.sh
+
+# invokes the changelog generator from
+# https://github.com/github-changelog-generator/github-changelog-generator
+#
+# With the config located in
+# arrow-datafusion/.github_changelog_generator
+#
+# Usage:
+# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log.sh
+
+set -e
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
+
+pushd ${SOURCE_TOP_DIR}
+docker run -it --rm \
+    -e CHANGELOG_GITHUB_TOKEN=$CHANGELOG_GITHUB_TOKEN \
+    -v "$(pwd)":/usr/local/src/your-app \
+    githubchangeloggenerator/github-changelog-generator \
+    --user apache \
+    --project arrow-datafusion \
+    --since-tag 4.0.0 \
+    --future-release 5.0.0
+
+sed -i "s/\\\n/\n\n/" CHANGELOG.md