[2.7] bpo-24960: use pkgutil.get_data in lib2to3 to read pickled gram… · python/cpython@770a802

File tree

5 files changed

lines changed

  • Misc/NEWS.d/next/Tools-Demos

5 files changed

lines changed

Original file line numberDiff line numberDiff line change

@@ -19,6 +19,7 @@

1919

import codecs

2020

import os

2121

import logging

22+

import pkgutil

2223

import StringIO

2324

import sys

2425

@@ -143,6 +144,26 @@ def _newer(a, b):

143144

return os.path.getmtime(a) >= os.path.getmtime(b)

144145
145146
147+

def load_packaged_grammar(package, grammar_source):

148+

"""Normally, loads a pickled grammar by doing

149+

pkgutil.get_data(package, pickled_grammar)

150+

where *pickled_grammar* is computed from *grammar_source* by adding the

151+

Python version and using a ``.pickle`` extension.

152+
153+

However, if *grammar_source* is an extant file, load_grammar(grammar_source)

154+

is called instead. This facilities using a packaged grammar file when needed

155+

but preserves load_grammar's automatic regeneration behavior when possible.

156+
157+

"""

158+

if os.path.isfile(grammar_source):

159+

return load_grammar(grammar_source)

160+

pickled_name = _generate_pickle_name(os.path.basename(grammar_source))

161+

data = pkgutil.get_data(package, pickled_name)

162+

g = grammar.Grammar()

163+

g.loads(data)

164+

return g

165+
166+
146167

def main(*args):

147168

"""Main program, when run as a script: produce grammar pickle files.

148169
Original file line numberDiff line numberDiff line change

@@ -109,6 +109,10 @@ def load(self, filename):

109109

f.close()

110110

self.__dict__.update(d)

111111
112+

def loads(self, pkl):

113+

"""Load the grammar tables from a pickle bytes object."""

114+

self.__dict__.update(pickle.loads(pkl))

115+
112116

def copy(self):

113117

"""

114118

Copy the grammar.

Original file line numberDiff line numberDiff line change

@@ -29,12 +29,12 @@ def __init__(self, grammar):

2929

setattr(self, name, symbol)

3030
3131
32-

python_grammar = driver.load_grammar(_GRAMMAR_FILE)

32+

python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE)

3333
3434

python_symbols = Symbols(python_grammar)

3535
3636

python_grammar_no_print_statement = python_grammar.copy()

3737

del python_grammar_no_print_statement.keywords["print"]

3838
39-

pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE)

39+

pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE)

4040

pattern_symbols = Symbols(pattern_grammar)

Original file line numberDiff line numberDiff line change

@@ -11,11 +11,14 @@

1111

from .support import driver, test_dir

1212
1313

# Python imports

14+

import operator

1415

import os

16+

import pickle

1517

import shutil

1618

import subprocess

1719

import sys

1820

import tempfile

21+

import types

1922

import unittest

2023
2124

# Local imports

@@ -97,6 +100,18 @@ def test_load_grammar_from_subprocess(self):

97100

finally:

98101

shutil.rmtree(tmpdir)

99102
103+

def test_load_packaged_grammar(self):

104+

modname = __name__ + '.load_test'

105+

class MyLoader:

106+

def get_data(self, where):

107+

return pickle.dumps({'elephant': 19})

108+

class MyModule(types.ModuleType):

109+

__file__ = 'parsertestmodule'

110+

__loader__ = MyLoader()

111+

sys.modules[modname] = MyModule(modname)

112+

self.addCleanup(operator.delitem, sys.modules, modname)

113+

g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')

114+

self.assertEqual(g.elephant, 19)

100115
101116
102117

class GrammarTest(support.TestCase):

Original file line numberDiff line numberDiff line change

@@ -0,0 +1,3 @@

1+

2to3 and lib2to3 can now read pickled grammar files using pkgutil.get_data()

2+

rather than probing the filesystem. This lets 2to3 and lib2to3 work when run

3+

from a zipfile.