From 304d1b3fdb8271cce68bf4e63faf43888b2e1f15 Mon Sep 17 00:00:00 2001 From: bendmorris Date: Fri, 11 Oct 2013 15:13:20 -0400 Subject: [PATCH 1/2] Initial commit. Re-creating repository as old repo was a fork of a bad mirror. --- README => PYTHON-README | 0 README.md | 99 +++++++++++ Static.make | 35 ++++ add_builtins.py | 352 ++++++++++++++++++++++++++++++++++++++++ static_freeze.py | 97 +++++++++++ 5 files changed, 583 insertions(+) rename README => PYTHON-README (100%) create mode 100644 README.md create mode 100644 Static.make create mode 100644 add_builtins.py create mode 100755 static_freeze.py diff --git a/README b/PYTHON-README similarity index 100% rename from README rename to PYTHON-README diff --git a/README.md b/README.md new file mode 100644 index 00000000000..62502b29e0f --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +Static-Python +============= + +This is a fork of the official Python hg repository with additional tools to +enable building Python for static linking. + +You may be wondering, "why would you want to do that? After all, [static linking +is evil](http://www.akkadia.org/drepper/no_static_linking.html)." Here are a few +possible reasons: + + * To run Python programs on other machines without requiring that they install + Python. + + * To run Python programs on other machines without requiring that they have + the same versions of the same libraries installed that you do. + + * Because the major binary distribution tools for Python (cx_Freeze, bbfreeze, + py2exe, and py2app) ship in a way that the Python source code can be + trivially derived (unzip the archive of .pyc files and decompile them.) For + proprietary or security-conscious applications, this is unacceptable. + + +Usage +===== + +Building Static Python +---------------------- + +To build a static Python executable and library, check out the appropriate branch +(either 2.7, 3.3, or master) and run the following command: + + make -f Static.make + +This will create an executable called `python` in the working directory, and a +static library, `libpythonX.X.a`, in the install/lib directory. You can confirm +that this executable is not dependent on any shared libraries using `ldd python` +which should report that python is not a dynamic executable. However, by default +this executable's functionality will be very limited - it won't even be able to +access most modules from the Python standard library. + +In order to make this Python interpreter truly standalone (not dependent on +installed Python modules), you can designate Python modules to be compiled as +builtins, which will be statically linked into the Python interpreter. +Static.make generates a file in Modules/Setup which needs to be edited to +specify these new builtin modules. + +You can automatically add builtins when building Static Python by passing +BUILTINS and/or SCRIPT variables to Static.make, e.g.: + + make -f Static.make BUILTINS="math zipfile zlib" SCRIPT="/path/to/script.py" + +Each module listed in the BUILTINS variable will be added, if possible. SCRIPT +can be used to specify the path to a Python script. This script will be scanned +for dependencies using modulefinder, and all dependencies will be added as +builtins if possible (not the script itself - it should be compiled using +static_freeze.py and linked to the resulting static library.) Finally, if the +DFLAG variable is set to "-d", all dependencies of all modules will be +automatically added as well (this will usually include many modules, and you may +not really need them all.) + +(If you previously built Static Python, you should `make -f Static.make clean` +first. Also, this step requires an existing Python installation, preferably of +the same version you're building, so you may need to build and install Python +the normal way first before building it statically..) + +Adding builtins can also be done manually by editing Modules/Setup. Add lines to +the end of the file in the format: + + module_name module.c ... + +These .c files can be generated from .py files using Cython: + + cython module.py + +(This is done automatically using add_builtins.py, the script called by +Static.make when BUILTINS or SCRIPT are supplied.) + +Packages are not currently supported. I'm working on an automatic solution for +compiling and including packages. Stay tuned. + + +Compile a standalone executable +------------------------------- + +Once you've compiled a static Python library, you can turn a Python script into +a standalone executable using the static_freeze.py script. + + Tools/static_freeze/static_freeze.py test.py libpython2.7.a + +This will generate an executable called "test" in the working directory, which +is not dependant on any shared libraries, Python modules, or the Python +interpreter! + + +Acknowledgements +================ + +Major thanks to Gabriel Jacobo who pioneered this method: + diff --git a/Static.make b/Static.make new file mode 100644 index 00000000000..93bbbcf266a --- /dev/null +++ b/Static.make @@ -0,0 +1,35 @@ +.PHONY: all setup clean + +all: python +setup: Modules/Setup + +PYTHON=python +PREFIX=$(shell pwd)/install +CONF_ARGS= +MAKE_ARGS= +#BUILTINS=array cmath math _struct time _operator _testcapi _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref +BUILTINS=atexit +SCRIPT= +DFLAG= +CPPFLAGS= +LDFLAGS= +INCLUDE=-I/usr/include + +Modules/Setup: Modules/Setup.dist add_builtins.py + sed -e 's/#\*shared\*/\*static\*/g' Modules/Setup.dist \ + > Modules/Setup + [ -d Modules/extras ] || mkdir Modules/extras + $(PYTHON) add_builtins.py $(BUILTINS) $(DFLAG) -s $(SCRIPT) + +Makefile: Modules/Setup + [ -d $(PREFIX) ] || mkdir $(PREFIX) + ./configure LDFLAGS="-Wl,-no-export-dynamic -static-libgcc -static $(LDFLAGS) $(INCLUDE)" \ + CPPFLAGS="-I/usr/lib -static -fPIC $(CPPLAGS) $(INCLUDE)" LINKFORSHARED=" " \ + DYNLOADFILE="dynload_stub.o" -disable-shared \ + -prefix="$(PREFIX)" $(CONF_ARGS) + +python: Modules/Setup Makefile + make $(MAKE_ARGS) + +clean: + rm -f Makefile Modules/Setup \ No newline at end of file diff --git a/add_builtins.py b/add_builtins.py new file mode 100644 index 00000000000..fae609d49a3 --- /dev/null +++ b/add_builtins.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python +'''Add a set of modules to the list of static Python builtins. + +Usage: python add_builtins.py [-s /path/to/script] [module_name] ... +''' +from modulefinder import ModuleFinder +import sys +import os +import os.path as op +import re +import imp +import shutil +import types +import importlib +from Cython.Build import cythonize +import Cython + + +# module definition lines in Modules/Setup look like this +module_def = re.compile('^[A-Za-z_\.]+ .+\.c') +# put extra builtins in here +extra_module_dir = op.join('Modules', 'extras') +if not op.exists(extra_module_dir): + os.makedirs(extra_module_dir) +# file endings that can be compiled by cython +compileable_exts = ('.c', '.cpp', 'module.c', 'module.cpp') + +def add_builtins(names, script=None, exclude=None, path=None, auto_add_deps=False, src_dirs=None): + if path is None: + paths = ['lib'] + sys.path + elif isinstance(path, basestring): + paths = [path, 'lib'] + sys.path + else: + paths = path + + if src_dirs is None: src_dirs = {} + + # if called with a script, find their dependencies and re-run + to_add = set(names) + if script: + module_dir = op.split(script)[0] + paths = [module_dir] + paths + finder = ModuleFinder(path=paths) + finder.run_script(script) + to_add.update(finder.modules.keys()) + return add_builtins(list(to_add), script=None, exclude=exclude, + path=paths, auto_add_deps=False, src_dirs=None) + + if auto_add_deps: + for name in names: + try: + f, module_path, _ = imp.find_module(name, paths) + except KeyboardInterrupt: raise + except: continue + + if any([module_path.endswith(x) for x in ('.py', '.pyc')]): + finder = ModuleFinder(path=paths) + finder.run_script(module_path) + to_add.update(finder.modules.keys()) + + + with open('Modules/Setup', 'r') as setup_file: + lines = [str(x).rstrip('\n') for x in setup_file.readlines()] + + # don't add sys (it's already builtin) or anything explicitly excluded + added = {'sys'} + if exclude: + added.update(exclude) + + + # check each module to see if a commented line is present in Modules/Setup, + # and uncomment + for n, line in enumerate(lines): + line = line.strip() + if not line: continue + + for name in to_add: + if line.startswith('#%s ' % name): + lines[n] = line.lstrip('#') + to_add.remove(name) + added.add(name) + print('** Added %s' % name) + break + + # keep track of uncommented module names in Modules/Setup + if module_def.match(line): + module_name = line.split()[0] + pkg = False + try: + f, module_path, _ = imp.find_module(module_name, paths) + if f is None: pkg = True + except: pkg = False + + if not pkg: + added.add(module_name) + print('** Found existing builtin %s' % module_name) + + # don't try to re-add existing builtins + to_add = set.difference(to_add, added) + + for name in list(to_add): + if name in added: continue + + new_lines = [] + + try: + f, module_path, _ = imp.find_module(name, paths) + except ImportError: + if '.' in name: f = None + else: + raise Exception("** Couldn't find module %s" % name) + continue + + # see if the target file already exists in Modules + search_paths = [op.join(*(search_dir + (name+x,))) + for x in compileable_exts + for search_dir in ( + (), + (name,), + ('extras', name), + ) + if op.exists(op.join(*(('Modules',) + search_dir + (name+x,)))) + ] if f else [] + + if search_paths: + module_file = search_paths[0] + print('** Added %s' % module_file) + else: + # import the target module using this python installation, + # and check the corresponding file + + if f is None: + # add package + pkg = name + print("*** Scanning package %s..." % pkg) + + try: + p = importlib.import_module(pkg) + except: + continue + + def get_submodules(x, yielded=None): + if not yielded: yielded = set() + yield x + yielded.add(x) + for member in dir(x): + member = getattr(x, member) + if isinstance(member, types.ModuleType) and not member in yielded: + for y in get_submodules(member, yielded): + yield y + yielded.add(y) + + submodules = get_submodules(p) + + for submodule in submodules: + name = submodule.__name__ + + sys.stdout.write("** Adding module %s in package %s..." % (name, pkg)) + sys.stdout.flush() + + try: + add = add_module(name, added, paths, src_dirs) + if add: new_lines += add + except Exception as e: + print(e) + + print('done.') + + else: + # add standalone module + sys.stdout.write('** Adding %s...' % name) + sys.stdout.flush() + + try: + add = add_module(name, added, paths, src_dirs, module_path=module_path) + if add: new_lines += add + except Exception as e: + print(e) + + print('done.') + + if new_lines: lines += new_lines + + with open('Modules/Setup', 'w') as setup_file: + setup_file.write('\n'.join(lines)) + + + +def add_module(name, added, paths, src_dirs, module_path=None): + if name in added: + return + + added.add(name) + pkg = '.' in name + opts = '' + + if not module_path: + try: module_path = importlib.import_module(name).__file__ + except: return + + # if it's a .pyc file, hope the original python source is right next to it! + if module_path.endswith('.pyc'): + if op.exists(module_path[:-1]): + module_path = module_path[:-1] + else: + raise Exception('Lone .pyc file %s' % module_path) + + module_dir, module_file = op.split(module_path) + + # copy the file to the Modules/extras directory + dest_dir = extra_module_dir + if pkg: + dest_dir = op.join(dest_dir, name.split('.')[0]) + if not op.exists(dest_dir): os.makedirs(dest_dir) + + + # if it's a shared library, try to find the original C or C++ + # source file to compile into a static library; otherwise, + # there's nothing we can do here + if module_file.endswith('.so'): + done = False + module_dirs = [] + for k, v in src_dirs.items(): + # if user specified a src directory for this package, + # include it in the module search path + if name.startswith(k): + module = name[len(k):].lstrip('.') + if '.' in module: + v = op.join(v, *module.split('.')) + module_dirs += [v] + module_dirs += [module_dir] + for search_dir in module_dirs + paths: + for compiled_module in ('.'.join(module_file.split('.')[:-1]) + ext + for ext in compileable_exts): + if op.exists(op.join(search_dir, compiled_module)): + dest_file = compiled_module + if pkg: + dest_file = '__'.join(name.split('.')) + '.' + dest_file.split('.')[-1] + + dest_path = op.join(dest_dir, dest_file) + + if not op.exists(dest_path): + shutil.copy(op.join(search_dir, compiled_module), dest_path) + + modile_dir = search_dir + module_file = dest_file + opts += ' -I%s' % op.abspath(search_dir) + done = True; break + + if done: break + + if not any([module_file.endswith(ext) for ext in compileable_exts]): + #raise Exception("Couldn't find C source file for %s" % module_file) + return + + module_path = op.join(module_dir, module_file) + + else: + # copy the file to the Modules/extras directory + dest_file = module_file + if pkg: + dest_file = '__'.join(name.split('.')) + '.' + dest_file.split('.')[-1] + + dest_path = op.join(dest_dir, dest_file) + opts = '' + + if not op.exists(dest_path): + shutil.copy(module_path, dest_path) + + + # if the file ends in .py or .pyx, try to compile with Cython + if any([module_file.endswith(x) for x in ('.py', '.pyx')]): + dest_file = '.'.join(dest_file.split('.')[:-1]) + '.c' + + if op.exists(op.join(dest_dir, dest_file)): + module_file = dest_file + else: + try: + cythonize(dest_path) + module_file = dest_file + dest_path = op.join(dest_dir, dest_file) + if pkg: + # correct module name in Cython-generated C file + wrong_name = '.'.join(dest_file.split('.')[:-1]) + print wrong_name, name, dest_path + + with open(dest_path, 'r') as input_file: + with open(dest_path + '2', 'w') as output_file: + for line in input_file: + line = line.replace('"%s"' % wrong_name, '"%s"' % name) + output_file.write(line) + + os.remove(dest_path) + shutil.move(dest_path + '2', dest_path) + + except KeyboardInterrupt: raise + except: + os.remove(op.join(dest_dir, dest_file)) + raise Exception('Cython failed to compile %s' % dest_path) + + if any([module_file.endswith(ext) for ext in compileable_exts]): + # if there's a directory called Modules/{module} or + # Modules/extras/{module}, include that directory when compiling + for inc_dir in (op.join('Modules', name), op.join(dest_dir, name)): + if op.exists(inc_dir) and op.isdir(inc_dir): + opts += ' -I%s' % op.abspath(inc_dir) + + if pkg: + module_file = op.join('extras', + name.split('.')[0], + '.'.join(module_file.split('.')[:-1]).replace( + '.', '__') + '.' + module_file.split('.')[-1] + ) + else: + module_file = op.join('extras', module_file) + + # add a line to Modules/Setup + return ['%s %s%s' % (name.replace('.', '__'), module_file, opts)] + else: + raise Exception('Unknown file: %s' % module_file) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + + parser.add_argument('module', nargs='*', help='names of modules to be added as builtins') + parser.add_argument('-s', '--script', nargs='?', default=None, + help='add all dependencies of this script as builtins') + parser.add_argument('-e', '--exclude', nargs='?', default=None, + help='comma-separated list of modules to be excluded') + parser.add_argument('-p', '--path', nargs='?', default=None, + help='add this path to the module search path') + parser.add_argument('-d', '--deps', action='store_true', + help='when adding a module, automatically add all of its dependencies') + parser.add_argument('--src', nargs='?', default=None, + help='list of source package locations for shared libraries, e.g. `pkg1:/path/to/src,pkg2:/path/to/src`') + + args = parser.parse_args() + + src_dirs = {pkg.split(':')[0]:pkg.split(':')[1] + for pkg in args.src.split(',') + } if args.src else None + + add_builtins(args.module, + script=args.script, + exclude=args.exclude.split(',') if args.exclude else None, + path=args.path, + auto_add_deps=args.deps, + src_dirs=src_dirs + ) diff --git a/static_freeze.py b/static_freeze.py new file mode 100755 index 00000000000..e555bfd4650 --- /dev/null +++ b/static_freeze.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +'''This script generates, runs, and then deletes a Makefile which will compile a +Python script into a standalone executable which is statically linked to the +Python runtime. + +To use this script, first generate a static Python library (see the parent +directory's README.md). + +Usage: + + python static_freeze.py test_file.py /path/to/static/libpythonX.X.a + +Any additional arguments will be passed to the generated Makefile, +e.g. PYTHON=python3 +''' + +import sys +import os +from subprocess import call +major_version = sys.version_info[0] + + +make_template = '''# Makefile for creating our standalone Cython program +PYTHON=./python +CYTHON=cython +PYVERSION=$(shell $(PYTHON) -c "import sys; print(sys.version[:3])") + +#INCDIR=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc())") +INCDIR=Include/ +PLATINCDIR=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc(plat_specific=True))") +LIBDIR1=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") +LIBDIR2=%library_dir% +PYLIB=%library_name% + +CC=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('CC'))") +LINKCC=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LINKCC'))") +LIBS=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LIBS'))") +SYSLIBS= $(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('SYSLIBS'))") + +INCLUDE= + +%name%: %name%.o %library_dir%/lib%library_name%.a + $(LINKCC) -o $@ $^ -L$() -L$(LIBDIR1) -L$(LIBDIR2) -l$(PYLIB) $(LIBS) $(SYSLIBS) $(INCLUDE) + +%name%.o: %name%.c + $(CC) -c $^ -I$(INCDIR) -I$(PLATINCDIR) $(INCLUDE) + +%name%.c: %filename% + @$(CYTHON) -%major% --embed %filename% + +all: %name% + +clean: + rm -f %name% %name%.c %name%.o +''' + +def freeze(filename, library, make_args=None): + if make_args is None: make_args = [] + + name = '.'.join(filename.split('.')[:-1]) + library_dir, library_name = os.path.split(os.path.abspath(library)) + library_name = '.'.join((library.split('/')[-1][3:]).split('.')[:-1]) + + template = make_template + # generate makefile + for a, b in ( + ('%name%', name), + ('%filename%', filename), + ('%library_dir%', library_dir), + ('%library_name%', library_name), + ('%major%', str(major_version)), + ): + template = template.replace(a, b) + + with open(filename + '.make', 'wb') as make_file: + make_file.write(bytes(template, 'utf8')) + + # call make + call(['make', '-f', '%s.make' % filename] + make_args) + + # delete makefile + os.remove(filename + '.make') + + +if __name__ == '__main__': + import sys + + try: script_file = sys.argv[1] + except: raise Exception('No script specified') + + try: lib_path = sys.argv[2] + except: raise Exception('Path to Python runtime not specified') + + try: make_args = sys.argv[3:] + except: make_args = [] + + freeze(script_file, lib_path, make_args=make_args) \ No newline at end of file From deba11d2909d979aaabb56c7f8e9f44bfb7151f0 Mon Sep 17 00:00:00 2001 From: bendmorris Date: Fri, 11 Oct 2013 15:30:10 -0400 Subject: [PATCH 2/2] Friendlier failures in static_freeze.py --- static_freeze.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/static_freeze.py b/static_freeze.py index e555bfd4650..2a3c2b2f7b0 100755 --- a/static_freeze.py +++ b/static_freeze.py @@ -83,15 +83,28 @@ def freeze(filename, library, make_args=None): if __name__ == '__main__': - import sys + def fail(message): + print(__doc__) + print(message) + sys.exit() - try: script_file = sys.argv[1] - except: raise Exception('No script specified') - - try: lib_path = sys.argv[2] - except: raise Exception('Path to Python runtime not specified') + try: + script_file = sys.argv[1] + assert os.path.exists(script_file) + except IndexError: + fail('ERROR: No script specified') + except AssertionError: + fail('ERROR: Script not found') + + try: + lib_path = sys.argv[2] + assert os.path.exists(lib_path) + except IndexError: + fail('ERROR: Path to Python runtime not specified') + except AssertionError: + fail('ERROR: Python runtime not found') try: make_args = sys.argv[3:] - except: make_args = [] + except IndexError: make_args = [] freeze(script_file, lib_path, make_args=make_args) \ No newline at end of file