Skip to content

Commit

Permalink
[close #772] Fix flaky integration tests (#770)
Browse files Browse the repository at this point in the history
* wait for tiup playground by grep

Signed-off-by: Ping Yu <yuping@pingcap.com>

* test on one version first

Signed-off-by: Ping Yu <yuping@pingcap.com>

* wip

Signed-off-by: Ping Yu <yuping@pingcap.com>

* reduce tikv resource usage

Signed-off-by: Ping Yu <yuping@pingcap.com>

* wip

Signed-off-by: Ping Yu <yuping@pingcap.com>

* wait for bootstrap

Signed-off-by: Ping Yu <yuping@pingcap.com>

* print logs on error

Signed-off-by: Ping Yu <yuping@pingcap.com>

* add api v2 test suite

Signed-off-by: Ping Yu <yuping@pingcap.com>

* use different port

Signed-off-by: Ping Yu <yuping@pingcap.com>

* specify kv.port

Signed-off-by: Ping Yu <yuping@pingcap.com>

* add more versions

Signed-off-by: Ping Yu <yuping@pingcap.com>

---------

Signed-off-by: Ping Yu <yuping@pingcap.com>
  • Loading branch information
pingyu authored Oct 7, 2023
1 parent cb26d58 commit 3b503fb
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 14 deletions.
11 changes: 7 additions & 4 deletions .github/config/tikv_rawkv.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,20 @@

[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "8G"
capacity = "6G"
pd-heartbeat-tick-interval = "2s"
pd-store-heartbeat-tick-interval = "5s"
split-region-check-tick-interval = "1s"

[storage]
enable-ttl = true

[rocksdb]
max-open-files = 10000

[raftdb]
max-open-files = 10000

[storage.block-cache]
capacity = "128MB"

[storage]
reserve-space = "0MB"
enable-ttl = true
8 changes: 7 additions & 1 deletion .github/config/tikv_txnkv.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

[raftstore]
# set store capacity, if no set, use disk capacity.
capacity = "8G"
capacity = "6G"
pd-heartbeat-tick-interval = "2s"
pd-store-heartbeat-tick-interval = "5s"
split-region-check-tick-interval = "1s"
Expand All @@ -12,3 +12,9 @@ max-open-files = 10000

[raftdb]
max-open-files = 10000

[storage.block-cache]
capacity = "128MB"

[storage]
reserve-space = "0MB"
17 changes: 17 additions & 0 deletions .github/config/tikv_v2.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# TiKV Configuration.

[raftstore]
pd-heartbeat-tick-interval = "2s"
pd-store-heartbeat-tick-interval = "5s"
split-region-check-tick-interval = "1s"

[rocksdb]
max-open-files = 10000

[raftdb]
max-open-files = 10000

[storage]
reserve-space = "0MB"
api-version = 2
enable-ttl = true
34 changes: 25 additions & 9 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
tikv_version: [nightly, v5.0.4, v5.3.0, v5.4.0]
tikv_version: [v5.0.6, v5.3.4, v5.4.3]
fail-fast: false
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
Expand All @@ -40,26 +41,41 @@ jobs:
java-version: '8.0'
distribution: 'adopt'
- name: Install TiUP
run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
/home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
- name: Start TiUP Playground
run: |
# Start TiKV in APIV1TTL
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 &
# The first run of `tiup` has to download all components so it'll take longer.
sleep 1m 30s
touch tiup-v1ttl.log
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag rawkv --mode tikv-slim --kv 1 --without-monitor --kv.port 20160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup-v1ttl.log &
timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1ttl.log)
cat tiup-v1ttl.log
echo "Wait for bootstrap"
sleep 10s
# Start TiKV in APIV1
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2381 2>&1 &
sleep 30s
touch tiup-v1.log
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag txnkv --mode tikv-slim --kv 1 --without-monitor --kv.port 30160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2381 2>&1 >> tiup-v1.log &
timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1.log)
cat tiup-v1.log
echo "Wait for bootstrap"
sleep 10s
# Get PD address
echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
echo "TXNKV_PD_ADDRESSES=127.0.0.1:2381" >> $GITHUB_ENV
- name: Run Integration Test
run: mvn clean test
- name: Print TiKV logs
if: failure()
run: |
echo "RawKV TiKV logs"
cat /home/runner/.tiup/data/rawkv/tikv-0/tikv.log
echo "TxnKV TiKV logs"
cat /home/runner/.tiup/data/txnkv/tikv-0/tikv.log
- name: Upload coverage
uses: codecov/codecov-action@v2
with:
Expand Down
52 changes: 52 additions & 0 deletions .github/workflows/ci_v2.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
name: CI (APIv2)

on:
pull_request:
push:
branches:
- master

jobs:
integration-test:
name: Integration Test - ${{ matrix.tikv_version }}
runs-on: ubuntu-latest
strategy:
matrix:
tikv_version: [v6.5.3, v7.1.1, nightly]
fail-fast: false
steps:
- uses: actions/checkout@v2
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
java-version: '8.0'
distribution: 'adopt'
- name: Install TiUP
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
/home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
- name: Start TiUP Playground
run: |
# Start TiKV in APIV2
touch tiup.log
/home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --tag kv --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_v2.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup.log &
timeout 300 grep -q "PD Endpoints:" <(tail -f tiup.log)
cat tiup.log
# Get PD address
echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
echo "TXNKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
- name: Run Integration Test
run: mvn clean test
- name: Print TiKV logs
if: failure()
run: |
echo "TiKV logs"
cat /home/runner/.tiup/data/kv/tikv-0/tikv.log
- name: Upload coverage
uses: codecov/codecov-action@v2
with:
files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
fail_ci_if_error: true
verbose: true

0 comments on commit 3b503fb

Please sign in to comment.