Home Original page

[BEAM-14429] Force java load test on dataflow runner v2 forceNumIniti… · tomstepp/apache-beam@178441d

6 files changed

lines changed

Original file line numberDiff line numberDiff line change

@@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder

2323

import CronJobBuilder

2424

import InfluxDBCredentialsHelper

2525
26+

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

27+
2628

def loadTestConfigurations = { mode, isStreaming ->

2729

[

2830

[

@@ -47,6 +49,7 @@ def loadTestConfigurations = { mode, isStreaming ->

4749

"numRecords": 20000000,

4850

"keySizeBytes": 10,

4951

"valueSizeBytes": 90,

52+

"forceNumInitialBundles": 1,

5053

"numHotKeys": 1

5154

}

5255

""".trim().replaceAll("\\s", ""),

@@ -55,6 +58,7 @@ def loadTestConfigurations = { mode, isStreaming ->

5558

"numRecords": 2000000,

5659

"keySizeBytes": 10,

5760

"valueSizeBytes": 90,

61+

"forceNumInitialBundles": 1,

5862

"numHotKeys": 1000

5963

}

6064

""".trim().replaceAll("\\s", ""),

@@ -86,6 +90,7 @@ def loadTestConfigurations = { mode, isStreaming ->

8690

"numRecords": 20000000,

8791

"keySizeBytes": 10,

8892

"valueSizeBytes": 90,

93+

"forceNumInitialBundles": 1,

8994

"numHotKeys": 5

9095

}

9196

""".trim().replaceAll("\\s", ""),

@@ -94,6 +99,7 @@ def loadTestConfigurations = { mode, isStreaming ->

9499

"numRecords": 2000000,

95100

"keySizeBytes": 10,

96101

"valueSizeBytes": 90,

102+

"forceNumInitialBundles": 1,

97103

"numHotKeys": 1000

98104

}

99105

""".trim().replaceAll("\\s", ""),

@@ -126,6 +132,7 @@ def loadTestConfigurations = { mode, isStreaming ->

126132

"numRecords": 2000000,

127133

"keySizeBytes": 10,

128134

"valueSizeBytes": 90,

135+

"forceNumInitialBundles": 1,

129136

"numHotKeys": 200000

130137

}

131138

""".trim().replaceAll("\\s", ""),

@@ -134,6 +141,7 @@ def loadTestConfigurations = { mode, isStreaming ->

134141

"numRecords": 2000000,

135142

"keySizeBytes": 10,

136143

"valueSizeBytes": 90,

144+

"forceNumInitialBundles": 1,

137145

"numHotKeys": 1000

138146

}

139147

""".trim().replaceAll("\\s", ""),

@@ -166,6 +174,7 @@ def loadTestConfigurations = { mode, isStreaming ->

166174

"numRecords": 2000000,

167175

"keySizeBytes": 10,

168176

"valueSizeBytes": 90,

177+

"forceNumInitialBundles": 1,

169178

"numHotKeys": 1000

170179

}

171180

""".trim().replaceAll("\\s", ""),

@@ -174,6 +183,7 @@ def loadTestConfigurations = { mode, isStreaming ->

174183

"numRecords": 2000000,

175184

"keySizeBytes": 10,

176185

"valueSizeBytes": 90,

186+

"forceNumInitialBundles": 1,

177187

"numHotKeys": 1000

178188

}

179189

""".trim().replaceAll("\\s", ""),

Original file line numberDiff line numberDiff line change

@@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder

2323

import CronJobBuilder

2424

import InfluxDBCredentialsHelper

2525
26+

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

27+
2628

def loadTestConfigurations = { mode, isStreaming ->

2729

[

2830

[

@@ -47,6 +49,7 @@ def loadTestConfigurations = { mode, isStreaming ->

4749

"numRecords": 20000000,

4850

"keySizeBytes": 10,

4951

"valueSizeBytes": 90,

52+

"forceNumInitialBundles": 1,

5053

"numHotKeys": 1

5154

}

5255

""".trim().replaceAll("\\s", ""),

@@ -55,6 +58,7 @@ def loadTestConfigurations = { mode, isStreaming ->

5558

"numRecords": 2000000,

5659

"keySizeBytes": 10,

5760

"valueSizeBytes": 90,

61+

"forceNumInitialBundles": 1,

5862

"numHotKeys": 1000

5963

}

6064

""".trim().replaceAll("\\s", ""),

@@ -86,6 +90,7 @@ def loadTestConfigurations = { mode, isStreaming ->

8690

"numRecords": 20000000,

8791

"keySizeBytes": 10,

8892

"valueSizeBytes": 90,

93+

"forceNumInitialBundles": 1,

8994

"numHotKeys": 5

9095

}

9196

""".trim().replaceAll("\\s", ""),

@@ -94,6 +99,7 @@ def loadTestConfigurations = { mode, isStreaming ->

9499

"numRecords": 2000000,

95100

"keySizeBytes": 10,

96101

"valueSizeBytes": 90,

102+

"forceNumInitialBundles": 1,

97103

"numHotKeys": 1000

98104

}

99105

""".trim().replaceAll("\\s", ""),

@@ -126,6 +132,7 @@ def loadTestConfigurations = { mode, isStreaming ->

126132

"numRecords": 2000000,

127133

"keySizeBytes": 10,

128134

"valueSizeBytes": 90,

135+

"forceNumInitialBundles": 1,

129136

"numHotKeys": 200000

130137

}

131138

""".trim().replaceAll("\\s", ""),

@@ -134,6 +141,7 @@ def loadTestConfigurations = { mode, isStreaming ->

134141

"numRecords": 2000000,

135142

"keySizeBytes": 10,

136143

"valueSizeBytes": 90,

144+

"forceNumInitialBundles": 1,

137145

"numHotKeys": 1000

138146

}

139147

""".trim().replaceAll("\\s", ""),

@@ -166,6 +174,7 @@ def loadTestConfigurations = { mode, isStreaming ->

166174

"numRecords": 2000000,

167175

"keySizeBytes": 10,

168176

"valueSizeBytes": 90,

177+

"forceNumInitialBundles": 1,

169178

"numHotKeys": 1000

170179

}

171180

""".trim().replaceAll("\\s", ""),

@@ -174,6 +183,7 @@ def loadTestConfigurations = { mode, isStreaming ->

174183

"numRecords": 2000000,

175184

"keySizeBytes": 10,

176185

"valueSizeBytes": 90,

186+

"forceNumInitialBundles": 1,

177187

"numHotKeys": 1000

178188

}

179189

""".trim().replaceAll("\\s", ""),

Original file line numberDiff line numberDiff line change

@@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder

2323

import CronJobBuilder

2424

import InfluxDBCredentialsHelper

2525
26+

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

27+
2628

def loadTestConfigurations = { mode, isStreaming ->

2729

[

2830

[

@@ -46,7 +48,8 @@ def loadTestConfigurations = { mode, isStreaming ->

4648

{

4749

"numRecords": 200000000,

4850

"keySizeBytes": 1,

49-

"valueSizeBytes": 9

51+

"valueSizeBytes": 9,

52+

"forceNumInitialBundles": 1

5053

}

5154

""".trim().replaceAll("\\s", ""),

5255

fanout : 1,

@@ -77,7 +80,8 @@ def loadTestConfigurations = { mode, isStreaming ->

7780

{

7881

"numRecords": 20000000,

7982

"keySizeBytes": 10,

80-

"valueSizeBytes": 90

83+

"valueSizeBytes": 90,

84+

"forceNumInitialBundles": 1

8185

}

8286

""".trim().replaceAll("\\s", ""),

8387

fanout : 1,

@@ -109,7 +113,8 @@ def loadTestConfigurations = { mode, isStreaming ->

109113

{

110114

"numRecords": 20000,

111115

"keySizeBytes": 10000,

112-

"valueSizeBytes": 90000

116+

"valueSizeBytes": 90000,

117+

"forceNumInitialBundles": 1

113118

}

114119

""".trim().replaceAll("\\s", ""),

115120

fanout : 1,

@@ -141,7 +146,8 @@ def loadTestConfigurations = { mode, isStreaming ->

141146

{

142147

"numRecords": 5000000,

143148

"keySizeBytes": 10,

144-

"valueSizeBytes": 90

149+

"valueSizeBytes": 90,

150+

"forceNumInitialBundles": 1

145151

}

146152

""".trim().replaceAll("\\s", ""),

147153

fanout : 4,

@@ -172,7 +178,8 @@ def loadTestConfigurations = { mode, isStreaming ->

172178

{

173179

"numRecords": 2500000,

174180

"keySizeBytes": 10,

175-

"valueSizeBytes": 90

181+

"valueSizeBytes": 90,

182+

"forceNumInitialBundles": 1

176183

}

177184

""".trim().replaceAll("\\s", ""),

178185

fanout : 8,

@@ -204,6 +211,7 @@ def loadTestConfigurations = { mode, isStreaming ->

204211

"numRecords": 20000000,

205212

"keySizeBytes": 10,

206213

"valueSizeBytes": 90,

214+

"forceNumInitialBundles": 1,

207215

"numHotKeys": 200,

208216

"hotKeyFraction": 1

209217

}

@@ -237,6 +245,7 @@ def loadTestConfigurations = { mode, isStreaming ->

237245

"numRecords": 20000000,

238246

"keySizeBytes": 10,

239247

"valueSizeBytes": 90,

248+

"forceNumInitialBundles": 1,

240249

"numHotKeys": 10,

241250

"hotKeyFraction": 1

242251

}

Original file line numberDiff line numberDiff line change

@@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder

2323

import CronJobBuilder

2424

import InfluxDBCredentialsHelper

2525
26+

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

27+
2628

def loadTestConfigurations = { mode, isStreaming ->

2729

[

2830

[

@@ -46,7 +48,8 @@ def loadTestConfigurations = { mode, isStreaming ->

4648

{

4749

"numRecords": 200000000,

4850

"keySizeBytes": 1,

49-

"valueSizeBytes": 9

51+

"valueSizeBytes": 9,

52+

"forceNumInitialBundles": 1

5053

}

5154

""".trim().replaceAll("\\s", ""),

5255

fanout : 1,

@@ -77,7 +80,8 @@ def loadTestConfigurations = { mode, isStreaming ->

7780

{

7881

"numRecords": 20000000,

7982

"keySizeBytes": 10,

80-

"valueSizeBytes": 90

83+

"valueSizeBytes": 90,

84+

"forceNumInitialBundles": 1

8185

}

8286

""".trim().replaceAll("\\s", ""),

8387

fanout : 1,

@@ -109,7 +113,8 @@ def loadTestConfigurations = { mode, isStreaming ->

109113

{

110114

"numRecords": 20000,

111115

"keySizeBytes": 10000,

112-

"valueSizeBytes": 90000

116+

"valueSizeBytes": 90000,

117+

"forceNumInitialBundles": 1

113118

}

114119

""".trim().replaceAll("\\s", ""),

115120

fanout : 1,

@@ -141,7 +146,8 @@ def loadTestConfigurations = { mode, isStreaming ->

141146

{

142147

"numRecords": 5000000,

143148

"keySizeBytes": 10,

144-

"valueSizeBytes": 90

149+

"valueSizeBytes": 90,

150+

"forceNumInitialBundles": 1

145151

}

146152

""".trim().replaceAll("\\s", ""),

147153

fanout : 4,

@@ -172,7 +178,8 @@ def loadTestConfigurations = { mode, isStreaming ->

172178

{

173179

"numRecords": 2500000,

174180

"keySizeBytes": 10,

175-

"valueSizeBytes": 90

181+

"valueSizeBytes": 90,

182+

"forceNumInitialBundles": 1

176183

}

177184

""".trim().replaceAll("\\s", ""),

178185

fanout : 8,

@@ -204,6 +211,7 @@ def loadTestConfigurations = { mode, isStreaming ->

204211

"numRecords": 20000000,

205212

"keySizeBytes": 10,

206213

"valueSizeBytes": 90,

214+

"forceNumInitialBundles": 1,

207215

"numHotKeys": 200,

208216

"hotKeyFraction": 1

209217

}

@@ -237,6 +245,7 @@ def loadTestConfigurations = { mode, isStreaming ->

237245

"numRecords": 20000000,

238246

"keySizeBytes": 10,

239247

"valueSizeBytes": 90,

248+

"forceNumInitialBundles": 1,

240249

"numHotKeys": 10,

241250

"hotKeyFraction": 1

242251

}

Original file line numberDiff line numberDiff line change

@@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder

2323

import CronJobBuilder

2424

import InfluxDBCredentialsHelper

2525
26+

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

27+
2628

def commonLoadTestConfig = { jobType, isStreaming ->

2729

[

2830

[

@@ -46,7 +48,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

4648

{

4749

"numRecords": 20000000,

4850

"keySizeBytes": 10,

49-

"valueSizeBytes": 90

51+

"valueSizeBytes": 90,

52+

"forceNumInitialBundles": 1

5053

}

5154

""".trim().replaceAll("\\s", ""),

5255

iterations : 10,

@@ -78,7 +81,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

7881

{

7982

"numRecords": 20000000,

8083

"keySizeBytes": 10,

81-

"valueSizeBytes": 90

84+

"valueSizeBytes": 90,

85+

"forceNumInitialBundles": 1

8286

}

8387

""".trim().replaceAll("\\s", ""),

8488

iterations : 200,

@@ -111,7 +115,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

111115

{

112116

"numRecords": 20000000,

113117

"keySizeBytes": 10,

114-

"valueSizeBytes": 90

118+

"valueSizeBytes": 90,

119+

"forceNumInitialBundles": 1

115120

}

116121

""".trim().replaceAll("\\s", ""),

117122

iterations : 1,

@@ -144,7 +149,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

144149

{

145150

"numRecords": 20000000,

146151

"keySizeBytes": 10,

147-

"valueSizeBytes": 90

152+

"valueSizeBytes": 90,

153+

"forceNumInitialBundles": 1

148154

}

149155

""".trim().replaceAll("\\s", ""),

150156

iterations : 1,

Original file line numberDiff line numberDiff line change

@@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder

2323

import CronJobBuilder

2424

import InfluxDBCredentialsHelper

2525
26+

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

27+
2628

def commonLoadTestConfig = { jobType, isStreaming ->

2729

[

2830

[

@@ -46,7 +48,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

4648

{

4749

"numRecords": 20000000,

4850

"keySizeBytes": 10,

49-

"valueSizeBytes": 90

51+

"valueSizeBytes": 90,

52+

"forceNumInitialBundles": 1

5053

}

5154

""".trim().replaceAll("\\s", ""),

5255

iterations : 10,

@@ -78,7 +81,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

7881

{

7982

"numRecords": 20000000,

8083

"keySizeBytes": 10,

81-

"valueSizeBytes": 90

84+

"valueSizeBytes": 90,

85+

"forceNumInitialBundles": 1

8286

}

8387

""".trim().replaceAll("\\s", ""),

8488

iterations : 200,

@@ -111,7 +115,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

111115

{

112116

"numRecords": 20000000,

113117

"keySizeBytes": 10,

114-

"valueSizeBytes": 90

118+

"valueSizeBytes": 90,

119+

"forceNumInitialBundles": 1

115120

}

116121

""".trim().replaceAll("\\s", ""),

117122

iterations : 1,

@@ -144,7 +149,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->

144149

{

145150

"numRecords": 20000000,

146151

"keySizeBytes": 10,

147-

"valueSizeBytes": 90

152+

"valueSizeBytes": 90,

153+

"forceNumInitialBundles": 1

148154

}

149155

""".trim().replaceAll("\\s", ""),

150156

iterations : 1,