bpo-40280: WASM docs and smaller browser builds (GH-32412) · python/cpython@defbbd6
@@ -20,7 +20,11 @@
2020SRCDIR_LIB = SRCDIR / "Lib"
21212222# sysconfig data relative to build dir.
23-SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
23+SYSCONFIGDATA = pathlib.PurePath(
24+"build",
25+f"lib.emscripten-wasm32-{sys.version_info.major}.{sys.version_info.minor}",
26+"_sysconfigdata__emscripten_wasm32-emscripten.py",
27+)
24282529# Library directory relative to $(prefix).
2630WASM_LIB = pathlib.PurePath("lib")
@@ -38,33 +42,44 @@
3842OMIT_FILES = (
3943# regression tests
4044"test/",
41-# user interfaces: TK, curses
42-"curses/",
43-"idlelib/",
44-"tkinter/",
45-"turtle.py",
46-"turtledemo/",
4745# package management
4846"ensurepip/",
4947"venv/",
5048# build system
5149"distutils/",
5250"lib2to3/",
53-# concurrency
54-"concurrent/",
55-"multiprocessing/",
5651# deprecated
5752"asyncore.py",
5853"asynchat.py",
59-# Synchronous network I/O and protocols are not supported; for example,
60-# socket.create_connection() raises an exception:
61-# "BlockingIOError: [Errno 26] Operation in progress".
54+"uu.py",
55+"xdrlib.py",
56+# other platforms
57+"_aix_support.py",
58+"_bootsubprocess.py",
59+"_osx_support.py",
60+# webbrowser
61+"antigravity.py",
62+"webbrowser.py",
63+# Pure Python implementations of C extensions
64+"_pydecimal.py",
65+"_pyio.py",
66+# Misc unused or large files
67+"pydoc_data/",
68+"msilib/",
69+)
70+71+# Synchronous network I/O and protocols are not supported; for example,
72+# socket.create_connection() raises an exception:
73+# "BlockingIOError: [Errno 26] Operation in progress".
74+OMIT_NETWORKING_FILES = (
6275"cgi.py",
6376"cgitb.py",
6477"email/",
6578"ftplib.py",
6679"http/",
6780"imaplib.py",
81+"mailbox.py",
82+"mailcap.py",
6883"nntplib.py",
6984"poplib.py",
7085"smtpd.py",
@@ -77,26 +92,28 @@
7792"urllib/response.py",
7893"urllib/robotparser.py",
7994"wsgiref/",
80-"xmlrpc/",
81-# dbm / gdbm
82-"dbm/",
83-# other platforms
84-"_aix_support.py",
85-"_bootsubprocess.py",
86-"_osx_support.py",
87-# webbrowser
88-"antigravity.py",
89-"webbrowser.py",
90-# ctypes
91-"ctypes/",
92-# Pure Python implementations of C extensions
93-"_pydecimal.py",
94-"_pyio.py",
95-# Misc unused or large files
96-"pydoc_data/",
97-"msilib/",
9895)
999697+OMIT_MODULE_FILES = {
98+"_asyncio": ["asyncio/"],
99+"audioop": ["aifc.py", "sunau.py", "wave.py"],
100+"_crypt": ["crypt.py"],
101+"_curses": ["curses/"],
102+"_ctypes": ["ctypes/"],
103+"_decimal": ["decimal.py"],
104+"_dbm": ["dbm/ndbm.py"],
105+"_gdbm": ["dbm/gnu.py"],
106+"_json": ["json/"],
107+"_multiprocessing": ["concurrent/", "multiprocessing/"],
108+"pyexpat": ["xml/", "xmlrpc/"],
109+"readline": ["rlcompleter.py"],
110+"_sqlite3": ["sqlite3/"],
111+"_ssl": ["ssl.py"],
112+"_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"],
113+114+"_zoneinfo": ["zoneinfo/"],
115+}
116+100117# regression test sub directories
101118OMIT_SUBDIRS = (
102119"ctypes/test/",
@@ -105,34 +122,59 @@
105122)
106123107124108-OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
109-OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
110-111-112-def filterfunc(name: str) -> bool:
113-return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
114-115-116125def create_stdlib_zip(
117-args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
126+args: argparse.Namespace,
127+*,
128+optimize: int = 0,
118129) -> None:
119-sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
120-if not sysconfig_data:
121-raise ValueError("No sysconfigdata file found")
130+def filterfunc(name: str) -> bool:
131+return not name.startswith(args.omit_subdirs_absolute)
122132123133with zipfile.PyZipFile(
124-args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
134+args.wasm_stdlib_zip, mode="w", compression=args.compression, optimize=optimize
125135 ) as pzf:
136+if args.compresslevel is not None:
137+pzf.compresslevel = args.compresslevel
138+pzf.writepy(args.sysconfig_data)
126139for entry in sorted(args.srcdir_lib.iterdir()):
127140if entry.name == "__pycache__":
128141continue
129-if entry in OMIT_ABSOLUTE:
142+if entry in args.omit_files_absolute:
130143continue
131144if entry.name.endswith(".py") or entry.is_dir():
132145# writepy() writes .pyc files (bytecode).
133146pzf.writepy(entry, filterfunc=filterfunc)
134-for entry in sysconfig_data:
135-pzf.writepy(entry)
147+148+149+def detect_extension_modules(args: argparse.Namespace):
150+modules = {}
151+152+# disabled by Modules/Setup.local ?
153+with open(args.builddir / "Makefile") as f:
154+for line in f:
155+if line.startswith("MODDISABLED_NAMES="):
156+disabled = line.split("=", 1)[1].strip().split()
157+for modname in disabled:
158+modules[modname] = False
159+break
160+161+# disabled by configure?
162+with open(args.sysconfig_data) as f:
163+data = f.read()
164+loc = {}
165+exec(data, globals(), loc)
166+167+for name, value in loc["build_time_vars"].items():
168+if value not in {"yes", "missing", "disabled", "n/a"}:
169+continue
170+if not name.startswith("MODULE_"):
171+continue
172+if name.endswith(("_CFLAGS", "_DEPS", "_LDFLAGS")):
173+continue
174+modname = name.removeprefix("MODULE_").lower()
175+if modname not in modules:
176+modules[modname] = value == "yes"
177+return modules
136178137179138180def path(val: str) -> pathlib.Path:
@@ -147,7 +189,10 @@ def path(val: str) -> pathlib.Path:
147189type=path,
148190)
149191parser.add_argument(
150-"--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
192+"--prefix",
193+help="install prefix",
194+default=pathlib.Path("/usr/local"),
195+type=path,
151196)
152197153198@@ -162,6 +207,27 @@ def main():
162207args.wasm_stdlib = args.wasm_root / WASM_STDLIB
163208args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
164209210+# bpo-17004: zipimport supports only zlib compression.
211+# Emscripten ZIP_STORED + -sLZ4=1 linker flags results in larger file.
212+args.compression = zipfile.ZIP_DEFLATED
213+args.compresslevel = 9
214+215+args.sysconfig_data = args.builddir / SYSCONFIGDATA
216+if not args.sysconfig_data.is_file():
217+raise ValueError(f"sysconfigdata file {SYSCONFIGDATA} missing.")
218+219+extmods = detect_extension_modules(args)
220+omit_files = list(OMIT_FILES)
221+omit_files.extend(OMIT_NETWORKING_FILES)
222+for modname, modfiles in OMIT_MODULE_FILES.items():
223+if not extmods.get(modname):
224+omit_files.extend(modfiles)
225+226+args.omit_files_absolute = {args.srcdir_lib / name for name in omit_files}
227+args.omit_subdirs_absolute = tuple(
228+str(args.srcdir_lib / name) for name in OMIT_SUBDIRS
229+ )
230+165231# Empty, unused directory for dynamic libs, but required for site initialization.
166232args.wasm_dynload.mkdir(parents=True, exist_ok=True)
167233marker = args.wasm_dynload / ".empty"
@@ -170,7 +236,7 @@ def main():
170236shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
171237# The rest of stdlib that's useful in a WASM context.
172238create_stdlib_zip(args)
173-size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
239+size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2)
174240parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
175241176242