Skip to content

Commit 87f712a

Browse files
feeblefakieKodaiD
andauthored
Backport to branch(3.17) : Fix the integration test and workflow for Object Storage adapter (#3242)
Co-authored-by: Kodai Doki <[email protected]>
1 parent e47567c commit 87f712a

10 files changed

+264
-24
lines changed

.github/workflows/object-storage-adapter-check.yaml

Lines changed: 120 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,25 +42,77 @@ env:
4242
INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"'
4343
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
4444
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
45-
S3_REGION: ap-northeast-1
46-
S3_BUCKET_NAME: scalardb-test-bucket
45+
S3_REGION: us-east-1
46+
S3_BUCKET_BASE_NAME: s3-scalardb-test-bucket
4747
CLOUD_STORAGE_PROJECT_ID: ${{ secrets.CLOUD_STORAGE_PROJECT_ID }}
4848
CLOUD_STORAGE_SERVICE_ACCOUNT_KEY: ${{ secrets.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}
49-
CLOUD_STORAGE_BUCKET_NAME: scalardb-test-bucket
49+
CLOUD_STORAGE_BUCKET_BASE_NAME: scalardb-test-bucket
5050

5151
jobs:
5252
integration-test-s3:
53-
name: S3 integration test (${{ matrix.mode.label }})
53+
name: S3 integration test (${{ matrix.test_group.label }})
5454
runs-on: ubuntu-latest
5555

5656
strategy:
5757
fail-fast: false
5858
matrix:
59-
mode:
60-
- label: default
59+
test_group:
60+
- label: consensus_commit_default
61+
tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"'
62+
bucket_suffix: consensus-commit
6163
group_commit_enabled: false
62-
- label: with_group_commit
64+
- label: consensus_commit_with_group_commit
65+
tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"'
66+
bucket_suffix: consensus-commit-gc
6367
group_commit_enabled: true
68+
- label: consensus_commit_admin
69+
tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"'
70+
bucket_suffix: consensus-commit-admin
71+
group_commit_enabled: false
72+
- label: consensus_commit_admin_with_group_commit
73+
tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"'
74+
bucket_suffix: consensus-commit-admin-gc
75+
group_commit_enabled: true
76+
- label: consensus_commit_specific
77+
tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"'
78+
bucket_suffix: consensus-commit-specific
79+
group_commit_enabled: false
80+
- label: consensus_commit_specific_with_group_commit
81+
tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"'
82+
bucket_suffix: consensus-commit-specific-gc
83+
group_commit_enabled: true
84+
- label: storage_scan_single
85+
tests_filter: '--tests "**.ObjectStorageSingle**"'
86+
bucket_suffix: storage-scan-single
87+
group_commit_enabled: false
88+
- label: storage_scan_multiple
89+
tests_filter: '--tests "**.ObjectStorageMultiple**"'
90+
bucket_suffix: storage-scan-multiple
91+
group_commit_enabled: false
92+
- label: storage_wrapper
93+
tests_filter: '--tests "**.ObjectStorageWrapper**"'
94+
bucket_suffix: storage-wrapper
95+
group_commit_enabled: false
96+
- label: storage_admin
97+
tests_filter: '--tests "**.ObjectStorageAdmin**"'
98+
bucket_suffix: storage-admin
99+
group_commit_enabled: false
100+
- label: storage_cm
101+
tests_filter: '--tests "**.ObjectStorageConditionalMutation**"'
102+
bucket_suffix: storage-cm
103+
group_commit_enabled: false
104+
- label: storage_others
105+
tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"'
106+
bucket_suffix: storage
107+
group_commit_enabled: false
108+
- label: two_phase_consensus_commit
109+
tests_filter: '--tests "**.TwoPhaseConsensusCommit**"'
110+
bucket_suffix: 2pcc
111+
group_commit_enabled: false
112+
- label: single_crud_operation_transaction
113+
tests_filter: '--tests "**.SingleCrudOperationTransaction**"'
114+
bucket_suffix: single-crud
115+
group_commit_enabled: false
64116

65117
steps:
66118
- uses: actions/checkout@v6
@@ -95,26 +147,79 @@ jobs:
95147
uses: gradle/actions/setup-gradle@v5
96148

97149
- name: Execute Gradle 'integrationTestObjectStorage' task
98-
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_NAME }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
150+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.AWS_ACCESS_KEY_ID }}' -Dscalardb.object_storage.password='${{ env.AWS_SECRET_ACCESS_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }}
99151

100152
- name: Upload Gradle test reports
101153
if: always()
102154
uses: actions/upload-artifact@v5
103155
with:
104-
name: s3_integration_test_reports_${{ matrix.mode.label }}
156+
name: s3_integration_test_reports_${{ matrix.test_group.label }}
105157
path: core/build/reports/tests/integrationTestObjectStorage
158+
106159
integration-test-cloud-storage:
107-
name: Cloud Storage integration test (${{ matrix.mode.label }})
160+
name: Cloud Storage integration test (${{ matrix.test_group.label }})
108161
runs-on: ubuntu-latest
109162

110163
strategy:
111164
fail-fast: false
112165
matrix:
113-
mode:
114-
- label: default
166+
test_group:
167+
- label: consensus_commit_default
168+
tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"'
169+
bucket_suffix: consensus-commit
115170
group_commit_enabled: false
116-
- label: with_group_commit
171+
- label: consensus_commit_with_group_commit
172+
tests_filter: '--tests "**.ConsensusCommitIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitNullMetadataIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitWithIncludeMetadataEnabledIntegrationTestWithObjectStorage"'
173+
bucket_suffix: consensus-commit-gc
117174
group_commit_enabled: true
175+
- label: consensus_commit_admin
176+
tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"'
177+
bucket_suffix: consensus-commit-admin
178+
group_commit_enabled: false
179+
- label: consensus_commit_admin_with_group_commit
180+
tests_filter: '--tests "**.ConsensusCommitAdminIntegrationTestWithObjectStorage" --tests "**.ConsensusCommitAdminRepairIntegrationTestWithObjectStorage"'
181+
bucket_suffix: consensus-commit-admin-gc
182+
group_commit_enabled: true
183+
- label: consensus_commit_specific
184+
tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"'
185+
bucket_suffix: consensus-commit-specific
186+
group_commit_enabled: false
187+
- label: consensus_commit_specific_with_group_commit
188+
tests_filter: '--tests "**.ConsensusCommitSpecificIntegrationTestWithObjectStorage"'
189+
bucket_suffix: consensus-commit-specific-gc
190+
group_commit_enabled: true
191+
- label: storage_scan_single
192+
tests_filter: '--tests "**.ObjectStorageSingle**"'
193+
bucket_suffix: storage-scan-single
194+
group_commit_enabled: false
195+
- label: storage_scan_multiple
196+
tests_filter: '--tests "**.ObjectStorageMultiple**"'
197+
bucket_suffix: storage-scan-multiple
198+
group_commit_enabled: false
199+
- label: storage_wrapper
200+
tests_filter: '--tests "**.ObjectStorageWrapper**"'
201+
bucket_suffix: storage-wrapper
202+
group_commit_enabled: false
203+
- label: storage_admin
204+
tests_filter: '--tests "**.ObjectStorageAdmin**"'
205+
bucket_suffix: storage-admin
206+
group_commit_enabled: false
207+
- label: storage_cm
208+
tests_filter: '--tests "**.ObjectStorageConditionalMutation**"'
209+
bucket_suffix: storage-cm
210+
group_commit_enabled: false
211+
- label: storage_others
212+
tests_filter: '--tests "**.ObjectStorageCaseSensitivity**" --tests "**.ObjectStorageColumnValue**" --tests "**.ObjectStorageCrossPartition**" --tests "**.ObjectStorageIntegrationTest" --tests "**.ObjectStorageJapanese**" --tests "**.ObjectStorageMutationAtomicity**" --tests "**.ObjectStorageWithReservedKeyword**"'
213+
bucket_suffix: storage
214+
group_commit_enabled: false
215+
- label: two_phase_consensus_commit
216+
tests_filter: '--tests "**.TwoPhaseConsensusCommit**"'
217+
bucket_suffix: 2pcc
218+
group_commit_enabled: false
219+
- label: single_crud_operation_transaction
220+
tests_filter: '--tests "**.SingleCrudOperationTransaction**"'
221+
bucket_suffix: single-crud
222+
group_commit_enabled: false
118223

119224
steps:
120225
- uses: actions/checkout@v6
@@ -149,11 +254,11 @@ jobs:
149254
uses: gradle/actions/setup-gradle@v5
150255

151256
- name: Execute Gradle 'integrationTestObjectStorage' task
152-
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=scalardb-test-bucket -Dscalardb.object_storage.username=${{ env.CLOUD_STORAGE_PROJECT_ID }} -Dscalardb.object_storage.password=${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
257+
run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=cloud-storage -Dscalardb.object_storage.endpoint=${{ env.CLOUD_STORAGE_BUCKET_BASE_NAME }}-${{ matrix.test_group.bucket_suffix }} -Dscalardb.object_storage.username='${{ env.CLOUD_STORAGE_PROJECT_ID }}' -Dscalardb.object_storage.password='${{ env.CLOUD_STORAGE_SERVICE_ACCOUNT_KEY }}' ${{ matrix.test_group.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }} ${{ matrix.test_group.tests_filter }}
153258

154259
- name: Upload Gradle test reports
155260
if: always()
156261
uses: actions/upload-artifact@v5
157262
with:
158-
name: cloud_storage_integration_test_reports_${{ matrix.mode.label }}
263+
name: cloud_storage_integration_test_reports_${{ matrix.test_group.label }}
159264
path: core/build/reports/tests/integrationTestObjectStorage

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,36 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.google.common.util.concurrent.Uninterruptibles;
34
import com.scalar.db.api.TableMetadata;
45
import com.scalar.db.io.DataType;
56
import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestBase;
67
import java.util.Properties;
8+
import java.util.concurrent.TimeUnit;
9+
import org.junit.jupiter.api.AfterEach;
10+
import org.junit.jupiter.api.BeforeEach;
711
import org.junit.jupiter.api.Disabled;
812

913
public class ConsensusCommitIntegrationTestWithObjectStorage
1014
extends ConsensusCommitIntegrationTestBase {
1115

16+
@Override
17+
@BeforeEach
18+
public void setUp() throws Exception {
19+
super.setUp();
20+
if (ObjectStorageEnv.isCloudStorage()) {
21+
// Sleep to mitigate rate limit errors
22+
Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
23+
}
24+
}
25+
26+
@AfterEach
27+
void tearDown() {
28+
if (ObjectStorageEnv.isCloudStorage()) {
29+
// Sleep to mitigate rate limit errors
30+
Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
31+
}
32+
}
33+
1234
@Override
1335
protected TableMetadata getTableMetadata() {
1436
return TableMetadata.newBuilder()

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitSpecificIntegrationTestWithObjectStorage.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,39 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.google.common.util.concurrent.Uninterruptibles;
34
import com.scalar.db.api.TableMetadata;
45
import com.scalar.db.io.DataType;
56
import com.scalar.db.transaction.consensuscommit.ConsensusCommitSpecificIntegrationTestBase;
67
import com.scalar.db.transaction.consensuscommit.Isolation;
78
import java.util.Properties;
9+
import java.util.concurrent.TimeUnit;
10+
import org.junit.jupiter.api.AfterEach;
11+
import org.junit.jupiter.api.BeforeEach;
812
import org.junit.jupiter.api.Disabled;
913

1014
public class ConsensusCommitSpecificIntegrationTestWithObjectStorage
1115
extends ConsensusCommitSpecificIntegrationTestBase {
1216

17+
@Override
18+
@BeforeEach
19+
protected void setUp() throws Exception {
20+
super.setUp();
21+
if (ObjectStorageEnv.isCloudStorage()) {
22+
// Sleep to mitigate rate limit errors
23+
Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
24+
}
25+
}
26+
27+
@Override
28+
@AfterEach
29+
public void tearDown() {
30+
super.tearDown();
31+
if (ObjectStorageEnv.isCloudStorage()) {
32+
// Sleep to mitigate rate limit errors
33+
Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS);
34+
}
35+
}
36+
1337
@Override
1438
protected TableMetadata getTableMetadata() {
1539
return TableMetadata.newBuilder()

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageConditionalMutationIntegrationTest.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@
66
public class ObjectStorageConditionalMutationIntegrationTest
77
extends DistributedStorageConditionalMutationIntegrationTestBase {
88

9-
@Override
10-
protected int getThreadNum() {
11-
return 3;
12-
}
13-
149
@Override
1510
protected Properties getProperties(String testName) {
1611
return ObjectStorageEnv.getProperties(testName);

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultipleClusteringKeyScanIntegrationTest.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageMultipleClusteringKeyScanIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageMultipleClusteringKeyScanIntegrationTest
712
extends DistributedStorageMultipleClusteringKeyScanIntegrationTestBase {
@@ -20,4 +25,15 @@ protected int getThreadNum() {
2025
protected boolean isParallelDdlSupported() {
2126
return false;
2227
}
28+
29+
@Override
30+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
31+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
32+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
33+
StringBuilder builder = new StringBuilder();
34+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
35+
return TextColumn.of(columnName, builder.toString());
36+
}
37+
return super.getColumnWithMaxValue(columnName, dataType);
38+
}
2339
}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageMultiplePartitionKeyIntegrationTest.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageMultiplePartitionKeyIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageMultiplePartitionKeyIntegrationTest
712
extends DistributedStorageMultiplePartitionKeyIntegrationTestBase {
@@ -20,4 +25,15 @@ protected int getThreadNum() {
2025
protected boolean isParallelDdlSupported() {
2126
return false;
2227
}
28+
29+
@Override
30+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
31+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
32+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
33+
StringBuilder builder = new StringBuilder();
34+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
35+
return TextColumn.of(columnName, builder.toString());
36+
}
37+
return super.getColumnWithMaxValue(columnName, dataType);
38+
}
2339
}
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,29 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageSingleClusteringKeyScanIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageSingleClusteringKeyScanIntegrationTest
712
extends DistributedStorageSingleClusteringKeyScanIntegrationTestBase {
13+
814
@Override
915
protected Properties getProperties(String testName) {
1016
return ObjectStorageEnv.getProperties(testName);
1117
}
18+
19+
@Override
20+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
21+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
22+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
23+
StringBuilder builder = new StringBuilder();
24+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
25+
return TextColumn.of(columnName, builder.toString());
26+
}
27+
return super.getColumnWithMaxValue(columnName, dataType);
28+
}
1229
}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,29 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageSinglePartitionKeyIntegrationTestBase;
4+
import com.scalar.db.io.Column;
5+
import com.scalar.db.io.DataType;
6+
import com.scalar.db.io.TextColumn;
7+
import com.scalar.db.util.TestUtils;
48
import java.util.Properties;
9+
import java.util.stream.IntStream;
510

611
public class ObjectStorageSinglePartitionKeyIntegrationTest
712
extends DistributedStorageSinglePartitionKeyIntegrationTestBase {
13+
814
@Override
915
protected Properties getProperties(String testName) {
1016
return ObjectStorageEnv.getProperties(testName);
1117
}
18+
19+
@Override
20+
protected Column<?> getColumnWithMaxValue(String columnName, DataType dataType) {
21+
if (dataType == DataType.TEXT && ObjectStorageEnv.isCloudStorage()) {
22+
// Since Cloud Storage can't handle 0xFF character correctly, we use "ZZZ..." as the max value
23+
StringBuilder builder = new StringBuilder();
24+
IntStream.range(0, TestUtils.MAX_TEXT_COUNT).forEach(i -> builder.append('Z'));
25+
return TextColumn.of(columnName, builder.toString());
26+
}
27+
return super.getColumnWithMaxValue(columnName, dataType);
28+
}
1229
}

0 commit comments

Comments
 (0)