Home Original page

[BEAM-12526] Add Dependabot (#17563) · tomstepp/apache-beam@fbc495c

@@ -120,121 +120,11 @@ def get_version():

120120

except ImportError:

121121

cythonize = lambda *args, **kwargs: []

122122123-

REQUIRED_PACKAGES = [

124-

# Avro 1.9.2 for python3 was broken. The issue was fixed in version 1.9.2.1

125-

'crcmod>=1.7,<2.0',

126-

# dataclasses backport for python_version<3.7. No version bound because this

127-

# is Python standard since Python 3.7 and each Python version is compatible

128-

# with a specific dataclasses version.

129-

'dataclasses;python_version<"3.7"',

130-

'orjson<4.0',

131-

# Dill doesn't have forwards-compatibility guarantees within minor version.

132-

# Pickles created with a new version of dill may not unpickle using older

133-

# version of dill. It is best to use the same version of dill on client and

134-

# server, therefore list of allowed versions is very narrow.

135-

# See: https://github.com/uqfoundation/dill/issues/341.

136-

'dill>=0.3.1.1,<0.3.2',

137-

'cloudpickle>=2.0.0,<3',

138-

'fastavro>=0.23.6,<2',

139-

'grpcio>=1.29.0,<2',

140-

'hdfs>=2.1.0,<3.0.0',

141-

'httplib2>=0.8,<0.21.0',

142-

'numpy>=1.14.3,<1.23.0',

143-

'pymongo>=3.8.0,<4.0.0',

144-

'protobuf>=3.12.2,<4',

145-

'proto-plus>=1.7.1,<2',

146-

'pyarrow>=0.15.1,<8.0.0',

147-

'pydot>=1.2.0,<2',

148-

'python-dateutil>=2.8.0,<3',

149-

'pytz>=2018.3',

150-

'requests>=2.24.0,<3.0.0',

151-

'typing-extensions>=3.7.0',

152-

]

153-154123

# [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.

155124

if sys.platform == 'win32' and sys.maxsize <= 2**32:

156-

REQUIRED_PACKAGES = [

157-

p for p in REQUIRED_PACKAGES if not p.startswith('pyarrow')

158-

]

159-160-

REQUIRED_TEST_PACKAGES = [

161-

'freezegun>=0.3.12',

162-

'joblib>=1.0.1',

163-

'mock>=1.0.1,<3.0.0',

164-

'pandas<2.0.0',

165-

'parameterized>=0.7.1,<0.8.0',

166-

'pyhamcrest>=1.9,!=1.10.0,<2.0.0',

167-

'pyyaml>=3.12,<7.0.0',

168-

'requests_mock>=1.7,<2.0',

169-

'tenacity>=5.0.2,<6.0',

170-

'pytest>=4.4.0,<5.0',

171-

'pytest-xdist>=1.29.0,<2',

172-

'pytest-timeout>=1.3.3,<2',

173-

'scikit-learn>=0.20.0',

174-

'sqlalchemy>=1.3,<2.0',

175-

'psycopg2-binary>=2.8.5,<3.0.0',

176-

'testcontainers[mysql]>=3.0.3,<4.0.0',

177-

'cryptography>=36.0.0',

178-

]

179-180-

GCP_REQUIREMENTS = [

181-

'cachetools>=3.1.0,<5',

182-

'google-apitools>=0.5.31,<0.5.32',

183-

# NOTE: Maintainers, please do not require google-auth>=2.x.x

184-

# Until this issue is closed

185-

# https://github.com/googleapis/google-cloud-python/issues/10566

186-

'google-auth>=1.18.0,<3',

187-

'google-auth-httplib2>=0.1.0,<0.2.0',

188-

'google-cloud-datastore>=1.8.0,<2',

189-

'google-cloud-pubsub>=2.1.0,<3',

190-

'google-cloud-pubsublite>=1.2.0,<2',

191-

# GCP packages required by tests

192-

'google-cloud-bigquery>=1.6.0,<3',

193-

'google-cloud-bigquery-storage>=2.6.3',

194-

'google-cloud-core>=0.28.1,<2',

195-

'google-cloud-bigtable>=0.31.1,<2',

196-

'google-cloud-spanner>=1.13.0,<2',

197-

'grpcio-gcp>=0.2.2,<1',

198-

# GCP Packages required by ML functionality

199-

'google-cloud-dlp>=3.0.0,<4',

200-

'google-cloud-language>=1.3.0,<2',

201-

'google-cloud-videointelligence>=1.8.0,<2',

202-

'google-cloud-vision>=0.38.0,<2',

203-

'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'

204-

]

205-206-

INTERACTIVE_BEAM = [

207-

'facets-overview>=1.0.0,<2',

208-

'google-cloud-dataproc>=3.0.0,<3.2.0',

209-

# IPython>=8 is not compatible with Python<=3.7

210-

'ipython>=7,<8;python_version<="3.7"',

211-

'ipython>=8,<9;python_version>"3.7"',

212-

'ipykernel>=6,<7',

213-

'ipywidgets>=7.6.5,<8',

214-

# Skip version 6.1.13 due to

215-

# https://github.com/jupyter/jupyter_client/issues/637

216-

'jupyter-client>=6.1.11,<6.1.13',

217-

'timeloop>=1.0.2,<2',

218-

]

219-220-

INTERACTIVE_BEAM_TEST = [

221-

# notebok utils

222-

'nbformat>=5.0.5,<6',

223-

'nbconvert>=6.2.0,<7',

224-

# headless chrome based integration tests

225-

'needle>=0.5.0,<1',

226-

'chromedriver-binary>=100,<101',

227-

# use a fixed major version of PIL for different python versions

228-

'pillow>=7.1.1,<8',

229-

]

230-231-

AWS_REQUIREMENTS = ['boto3 >=1.9']

232-233-

AZURE_REQUIREMENTS = [

234-

'azure-storage-blob >=12.3.2',

235-

'azure-core >=1.7.0',

236-

]

237-125+

pyarrow_dependency = ''

126+

else:

127+

pyarrow_dependency = 'pyarrow>=0.15.1,<8.0.0'

238128239129

# We must generate protos after setup_requires are installed.

240130

def generate_protos_first():

@@ -272,6 +162,8 @@ def get_portability_package_data():

272162

# structure must exist before the call to setuptools.find_packages()

273163

# executes below.

274164

generate_protos_first()

165+

# Keep all dependencies inlined in the setup call, otherwise Dependabot won't

166+

# be able to parse it.

275167

setuptools.setup(

276168

name=PACKAGE_NAME,

277169

version=PACKAGE_VERSION,

@@ -309,7 +201,39 @@ def get_portability_package_data():

309201

'apache_beam/utils/counters.py',

310202

'apache_beam/utils/windowed_value.py',

311203

]),

312-

install_requires=REQUIRED_PACKAGES,

204+

install_requires=[

205+

# Avro 1.9.2 for python3 was broken.

206+

# The issue was fixed in version 1.9.2.1

207+

'crcmod>=1.7,<2.0',

208+

# dataclasses backport for python_version<3.7. No version bound because

209+

# this is Python standard since Python 3.7 and each Python version is

210+

# compatible with a specific dataclasses version.

211+

'dataclasses;python_version<"3.7"',

212+

'orjson<4.0',

213+

# Dill doesn't have forwards-compatibility guarantees within minor

214+

# version. Pickles created with a new version of dill may not unpickle

215+

# using older version of dill. It is best to use the same version of

216+

# dill on client and server, therefore list of allowed versions is very

217+

# narrow. See: https://github.com/uqfoundation/dill/issues/341.

218+

'dill>=0.3.1.1,<0.3.2',

219+

'cloudpickle>=2.0.0,<3',

220+

'fastavro>=0.23.6,<2',

221+

'grpcio>=1.29.0,<2',

222+

'hdfs>=2.1.0,<3.0.0',

223+

'httplib2>=0.8,<0.21.0',

224+

'numpy>=1.14.3,<1.23.0',

225+

'pymongo>=3.8.0,<4.0.0',

226+

'protobuf>=3.12.2,<4',

227+

'proto-plus>=1.7.1,<2',

228+

'pydot>=1.2.0,<2',

229+

'python-dateutil>=2.8.0,<3',

230+

'pytz>=2018.3',

231+

'requests>=2.24.0,<3.0.0',

232+

'typing-extensions>=3.7.0',

233+

# Dynamic dependencies must be specified in a separate list, otherwise

234+

# Dependabot won't be able to parse the main list. Any dynamic

235+

# dependencies will not receive updates from Dependabot.

236+

] + [pyarrow_dependency],

313237

python_requires=python_requires,

314238

# BEAM-8840: Do NOT use tests_require or setup_requires.

315239

extras_require={

@@ -319,12 +243,78 @@ def get_portability_package_data():

319243

# https://github.com/sphinx-doc/sphinx/issues/9727

320244

'docutils==0.17.1'

321245

],

322-

'test': REQUIRED_TEST_PACKAGES,

323-

'gcp': GCP_REQUIREMENTS,

324-

'interactive': INTERACTIVE_BEAM,

325-

'interactive_test': INTERACTIVE_BEAM_TEST,

326-

'aws': AWS_REQUIREMENTS,

327-

'azure': AZURE_REQUIREMENTS,

246+

'test': [

247+

'freezegun>=0.3.12',

248+

'joblib>=1.0.1',

249+

'mock>=1.0.1,<3.0.0',

250+

'pandas<2.0.0',

251+

'parameterized>=0.7.1,<0.8.0',

252+

'pyhamcrest>=1.9,!=1.10.0,<2.0.0',

253+

'pyyaml>=3.12,<7.0.0',

254+

'requests_mock>=1.7,<2.0',

255+

'tenacity>=5.0.2,<6.0',

256+

'pytest>=4.4.0,<5.0',

257+

'pytest-xdist>=1.29.0,<2',

258+

'pytest-timeout>=1.3.3,<2',

259+

'scikit-learn>=0.20.0',

260+

'sqlalchemy>=1.3,<2.0',

261+

'psycopg2-binary>=2.8.5,<3.0.0',

262+

'testcontainers[mysql]>=3.0.3,<4.0.0',

263+

'cryptography>=36.0.0',

264+

],

265+

'gcp': [

266+

'cachetools>=3.1.0,<5',

267+

'google-apitools>=0.5.31,<0.5.32',

268+

# NOTE: Maintainers, please do not require google-auth>=2.x.x

269+

# Until this issue is closed

270+

# https://github.com/googleapis/google-cloud-python/issues/10566

271+

'google-auth>=1.18.0,<3',

272+

'google-auth-httplib2>=0.1.0,<0.2.0',

273+

'google-cloud-datastore>=1.8.0,<2',

274+

'google-cloud-pubsub>=2.1.0,<3',

275+

'google-cloud-pubsublite>=1.2.0,<2',

276+

# GCP packages required by tests

277+

'google-cloud-bigquery>=1.6.0,<3',

278+

'google-cloud-bigquery-storage>=2.6.3',

279+

'google-cloud-core>=0.28.1,<2',

280+

'google-cloud-bigtable>=0.31.1,<2',

281+

'google-cloud-spanner>=1.13.0,<2',

282+

'grpcio-gcp>=0.2.2,<1',

283+

# GCP Packages required by ML functionality

284+

'google-cloud-dlp>=3.0.0,<4',

285+

'google-cloud-language>=1.3.0,<2',

286+

'google-cloud-videointelligence>=1.8.0,<2',

287+

'google-cloud-vision>=0.38.0,<2',

288+

'google-cloud-recommendations-ai>=0.1.0,<=0.2.0'

289+

],

290+

'interactive': [

291+

'facets-overview>=1.0.0,<2',

292+

'google-cloud-dataproc>=3.0.0,<3.2.0',

293+

# IPython>=8 is not compatible with Python<=3.7

294+

'ipython>=7,<8;python_version<="3.7"',

295+

'ipython>=8,<9;python_version>"3.7"',

296+

'ipykernel>=6,<7',

297+

'ipywidgets>=7.6.5,<8',

298+

# Skip version 6.1.13 due to

299+

# https://github.com/jupyter/jupyter_client/issues/637

300+

'jupyter-client>=6.1.11,<6.1.13',

301+

'timeloop>=1.0.2,<2',

302+

],

303+

'interactive_test': [

304+

# notebok utils

305+

'nbformat>=5.0.5,<6',

306+

'nbconvert>=6.2.0,<7',

307+

# headless chrome based integration tests

308+

'needle>=0.5.0,<1',

309+

'chromedriver-binary>=100,<101',

310+

# use a fixed major version of PIL for different python versions

311+

'pillow>=7.1.1,<8',

312+

],

313+

'aws': ['boto3 >=1.9'],

314+

'azure': [

315+

'azure-storage-blob >=12.3.2',

316+

'azure-core >=1.7.0',

317+

],

328318

'dataframe': ['pandas>=1.0,<1.5']

329319

},

330320

zip_safe=False,