Home Original page

bpo-45019: Cleanup module freezing and deepfreeze (#29772) · python/cpython@b0b10e1

@@ -8,22 +8,20 @@

88

import os

99

import ntpath

1010

import posixpath

11-

import platform

12-

import subprocess

1311

import sys

14-

import time

151216-

from update_file import updating_file_with_tmpfile, update_file_with_tmpfile

13+

from update_file import updating_file_with_tmpfile

171418151916

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))

2017

ROOT_DIR = os.path.abspath(ROOT_DIR)

2118

FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')

22192320

STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')

24-

# If MODULES_DIR is changed then the .gitattributes and .gitignore files

25-

# need to be updated.

26-

MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')

21+

# If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the

22+

# .gitattributes and .gitignore files needs to be updated.

23+

FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')

24+

DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')

27252826

FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')

2927

MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')

@@ -111,16 +109,16 @@ def relpath_for_posix_display(path, base):

111109

#######################################

112110

# specs

113111114-

def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):

112+

def parse_frozen_specs():

115113

seen = {}

116-

for section, specs in sectionalspecs:

114+

for section, specs in FROZEN:

117115

parsed = _parse_specs(specs, section, seen)

118116

for item in parsed:

119117

frozenid, pyfile, modname, ispkg, section = item

120118

try:

121119

source = seen[frozenid]

122120

except KeyError:

123-

source = FrozenSource.from_id(frozenid, pyfile, destdir)

121+

source = FrozenSource.from_id(frozenid, pyfile)

124122

seen[frozenid] = source

125123

else:

126124

assert not pyfile or pyfile == source.pyfile, item

@@ -225,15 +223,16 @@ def iter_subs():

225223

#######################################

226224

# frozen source files

227225228-

class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):

226+

class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):

229227230228

@classmethod

231-

def from_id(cls, frozenid, pyfile=None, destdir=MODULES_DIR):

229+

def from_id(cls, frozenid, pyfile=None):

232230

if not pyfile:

233231

pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'

234232

#assert os.path.exists(pyfile), (frozenid, pyfile)

235-

frozenfile = resolve_frozen_file(frozenid, destdir)

236-

return cls(frozenid, pyfile, frozenfile)

233+

frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)

234+

deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR)

235+

return cls(frozenid, pyfile, frozenfile, deepfreezefile)

237236238237

@property

239238

def frozenid(self):

@@ -261,7 +260,7 @@ def ispkg(self):

261260

return os.path.basename(self.pyfile) == '__init__.py'

262261263262264-

def resolve_frozen_file(frozenid, destdir=MODULES_DIR):

263+

def resolve_frozen_file(frozenid, destdir):

265264

"""Return the filename corresponding to the given frozen ID.

266265267266

For stdlib modules the ID will always be the full name

@@ -570,41 +569,30 @@ def regen_makefile(modules):

570569

deepfreezefiles = []

571570

rules = ['']

572571

deepfreezerules = ['']

573-574-

# TODO: Merge the two loops

575572

for src in _iter_sources(modules):

576-

header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)

577-

relfile = header.replace('\\', '/')

578-

_pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)

579-580-

# TODO: This is a bit hackish

581-

xfile = relfile.replace("/frozen_modules/", "/deepfreeze/")

582-

cfile = xfile[:-2] + ".c"

583-

ofile = xfile[:-2] + ".o"

573+

frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)

574+

deepfreeze_header = relpath_for_posix_display(src.deepfreezefile, ROOT_DIR)

575+

frozenfiles.append(f'\t\t{frozen_header} \\')

576+

cfile = deepfreeze_header[:-2] + ".c"

577+

ofile = deepfreeze_header[:-2] + ".o"

584578

deepfreezefiles.append(f"\t\t{ofile} \\")

585579586-

# Also add a deepfreeze rule.

587-

deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)')

588-

deepfreezerules.append(

589-

f"\t$(PYTHON_FOR_REGEN) "

590-

f"$(srcdir)/Tools/scripts/deepfreeze.py "

591-

f"{header} -m {src.frozenid} -o {cfile}")

592-

deepfreezerules.append('')

593-594-

for src in _iter_sources(modules):

595-

header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)

596-

frozenfiles.append(f'\t\t{header} \\')

597-598580

pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)

599581

pyfiles.append(f'\t\t{pyfile} \\')

600582601583

freeze = (f'$(FREEZE_MODULE) {src.frozenid} '

602-

f'$(srcdir)/{pyfile} {header}')

584+

f'$(srcdir)/{pyfile} {frozen_header}')

603585

rules.extend([

604-

f'{header}: $(FREEZE_MODULE) {pyfile}',

586+

f'{frozen_header}: $(FREEZE_MODULE) {pyfile}',

605587

f'\t{freeze}',

606588

'',

607589

])

590+

deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')

591+

deepfreezerules.append(

592+

f"\t$(PYTHON_FOR_REGEN) "

593+

f"$(srcdir)/Tools/scripts/deepfreeze.py "

594+

f"{frozen_header} -m {src.frozenid} -o {cfile}")

595+

deepfreezerules.append('')

608596

pyfiles[-1] = pyfiles[-1].rstrip(" \\")

609597

frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")

610598

deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\")

@@ -715,7 +703,7 @@ def regen_pcbuild(modules):

715703716704

def main():

717705

# Expand the raw specs, preserving order.

718-

modules = list(parse_frozen_specs(destdir=MODULES_DIR))

706+

modules = list(parse_frozen_specs())

719707720708

# Regen build-related files.

721709

regen_makefile(modules)