From ec1ff4ca822103cf0f0fdeda17fb1dfbfdb13e89 Mon Sep 17 00:00:00 2001 From: RonaldsonBellande Date: Tue, 22 Jul 2025 18:40:32 -0400 Subject: [PATCH] first commit --- README.md | 236 +++++++++++++++ __init__.py | 0 git_scripts/.gitignore | 3 + header_imports.py | 9 + header_imports/__init__.py | 0 .../header_imports_initialization.py | 5 + .../header_imports_python_library.py | 3 + main.py | 88 ++++++ scripts/bellande_python_executable_test.sh | 5 + scripts/publish.sh | 2 + setup.py | 0 src/.gitignore | 1 + src/__init__.py | 0 src/analyzer.py | 203 +++++++++++++ src/builder.py | 278 ++++++++++++++++++ src/collector.py | 233 +++++++++++++++ src/compiler.py | 122 ++++++++ src/utilities.py | 199 +++++++++++++ test_files/test.py | 1 + 19 files changed, 1388 insertions(+) create mode 100644 README.md create mode 100644 __init__.py create mode 100644 git_scripts/.gitignore create mode 100644 header_imports.py create mode 100644 header_imports/__init__.py create mode 100644 header_imports/header_imports_initialization.py create mode 100644 header_imports/header_imports_python_library.py create mode 100644 main.py create mode 100755 scripts/bellande_python_executable_test.sh create mode 100755 scripts/publish.sh create mode 100644 setup.py create mode 100644 src/.gitignore create mode 100644 src/__init__.py create mode 100644 src/analyzer.py create mode 100644 src/builder.py create mode 100644 src/collector.py create mode 100644 src/compiler.py create mode 100644 src/utilities.py create mode 100644 test_files/test.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..d1e9e6b --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# Bellande Python Executable + +bellande_python_executable is a Python tool that converts Python scripts into standalone executables, similar to PyInstaller but with a simpler architecture and focus on multi-platform operations. + +## Features + +- Convert Python scripts to native executables +- Automatic dependency analysis +- Support for standard library, third-party, and local modules +- Single-file executable output +- Cross-platform support (Linux, Windows, macOS) +- No external dependencies required +- Debug mode for troubleshooting + +## Installation + +```bash +git clone +cd bellande_python_executable +pip install -e . +``` + +## Requirements + +- Python 3.7 or higher +- C compiler (GCC on Linux/macOS, MSVC or GCC on Windows) +- Python development headers + +### Installing Requirements on Linux + +```bash +# Ubuntu/Debian +sudo apt-get install python3-dev gcc + +# CentOS/RHEL/Fedora +sudo dnf install python3-devel gcc + +# Or for older versions +sudo yum install python3-devel gcc +``` + +### Installing Requirements on Windows + +- Install Visual Studio Build Tools or Visual Studio Community +- Or install GCC via MinGW-w64 + +### Installing Requirements on macOS + +```bash +xcode-select --install +``` + +## Usage + +### Basic Usage + +```bash +bellande_python_executable script.py +``` + +This will create an executable named `script` (or `script.exe` on Windows) in the `dist/` directory. + +### Advanced Usage + +```bash +bellande_python_executable script.py \ + --output myapp \ + --onefile \ + --exclude tkinter \ + --include requests \ + --add-data "data.txt:." \ + --debug +``` + +### Command Line Options + +- `script` - Python script to convert (required) +- `-o, --output` - Output executable name +- `-n, --name` - Name of the executable +- `--onefile` - Create a single executable file (default) +- `--windowed` - Create windowed application (no console) +- `--debug` - Enable debug mode +- `--exclude` - Exclude modules (can be used multiple times) +- `--include` - Include additional modules (can be used multiple times) +- `--add-data` - Add data files in format `src:dest` (can be used multiple times) + +## Examples + +### Simple Script + +```python +# hello.py +print("Hello, World!") +``` + +```bash +bellande_python_executable hello.py +./dist/hello +``` + +### Script with Dependencies + +```python +# web_scraper.py +import requests +from bs4 import BeautifulSoup + +def main(): + response = requests.get("https://httpbin.org/json") + print(response.json()) + +if __name__ == "__main__": + main() +``` + +```bash +bellande_python_executable web_scraper.py --include requests --include bs4 +./dist/web_scraper +``` + +### Script with Data Files + +```python +# config_app.py +import json + +def main(): + with open("config.json", "r") as f: + config = json.load(f) + print(f"App name: {config['name']}") + +if __name__ == "__main__": + main() +``` + +```bash +bellande_python_executable config_app.py --add-data "config.json:." +./dist/config_app +``` + +## Architecture + +bellande_python_executable consists of several modules: + +1. **main.py** - Entry point and command-line interface +2. **analyzer.py** - Dependency analysis using AST parsing +3. **collector.py** - Code and resource collection +4. **compiler.py** - Bytecode compilation and archiving +5. **builder.py** - Executable generation with C bootstrap +6. **utils.py** - Utility functions and configuration management + +### Build Process + +1. **Analysis Phase** - Analyze the main script and discover all dependencies +2. **Collection Phase** - Gather all required Python files and resources +3. **Compilation Phase** - Compile Python source to bytecode and create archives +4. **Building Phase** - Generate C bootstrap code and compile to executable + +## How It Works + +bellande_python_executable creates a C executable that: + +1. Embeds Python bytecode and resources as binary data +2. Initializes the Python interpreter at runtime +3. Loads and executes the embedded bytecode +4. Provides a custom import system for bundled modules + +The generated executable is completely self-contained and doesn't require Python to be installed on the target system. + +## Troubleshooting + +### Common Issues + +1. **Missing Python headers** + ``` + error: Python.h: No such file or directory + ``` + Solution: Install Python development packages (python3-dev on Ubuntu) + +2. **Compiler not found** + ``` + gcc: command not found + ``` + Solution: Install GCC or appropriate C compiler + +3. **Import errors in generated executable** + ``` + ModuleNotFoundError: No module named 'xyz' + ``` + Solution: Use `--include xyz` to explicitly include the module + +### Debug Mode + +Use `--debug` flag to enable verbose logging: + +```bash +bellande_python_executable script.py --debug +``` + +This will show detailed information about: +- Discovered dependencies +- Collected files +- Compilation process +- Build steps + +## Limitations + +- Requires C compiler on build system +- Some dynamic imports may not be detected automatically +- Binary size may be large due to embedded Python runtime +- Limited support for Python extensions that require specific loading mechanisms + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests if applicable +5. Submit a pull request + +## License + +This project is licensed under the MIT License - see the LICENSE file for details. + +## Comparison with PyInstaller + +| Feature | bellande_python_executable | PyInstaller | +|---------|----------|-------------| +| Dependencies | None | Multiple | +| Build system | Simple C bootstrap | Complex bundling | +| Size | Moderate | Smaller | +| Compatibility | Good | Excellent | +| Customization | High | Moderate | +| Learning curve | Low | Moderate | + +bellande_python_executable is designed to be a simpler, more transparent alternative to PyInstaller with fewer dependencies and easier customization. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/git_scripts/.gitignore b/git_scripts/.gitignore new file mode 100644 index 0000000..e5a7a9c --- /dev/null +++ b/git_scripts/.gitignore @@ -0,0 +1,3 @@ +fix_errors.sh +push.sh +repository_recal.sh diff --git a/header_imports.py b/header_imports.py new file mode 100644 index 0000000..974001f --- /dev/null +++ b/header_imports.py @@ -0,0 +1,9 @@ +# Contain Everything +import sys +sys.path.append("header_imports/") + +# Header Initialization +from header_imports_python_library import * + +# Header Initialization +from header_imports_initialization import * diff --git a/header_imports/__init__.py b/header_imports/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/header_imports/header_imports_initialization.py b/header_imports/header_imports_initialization.py new file mode 100644 index 0000000..0d5fb5c --- /dev/null +++ b/header_imports/header_imports_initialization.py @@ -0,0 +1,5 @@ +from analyzer import * +from collector import * +from compiler import * +from builder import * +from utilities import * diff --git a/header_imports/header_imports_python_library.py b/header_imports/header_imports_python_library.py new file mode 100644 index 0000000..cf8732a --- /dev/null +++ b/header_imports/header_imports_python_library.py @@ -0,0 +1,3 @@ +import argparse, importlib.util, sys, os, ast, shutil, time, py_compile, marshal, zipfile, subprocess, tempfile +from pathlib import Path +from typing import Set, List, Dict, Optional diff --git a/main.py b/main.py new file mode 100644 index 0000000..61e2383 --- /dev/null +++ b/main.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +PyPack - A Python to executable converter +Main entry point for the application +""" +from header_imports import * + +def main(): + parser = argparse.ArgumentParser(description='Convert Python scripts to executables') + parser.add_argument('-script', help='Python script or file to convert') + parser.add_argument('-o', '--output', help='Output executable name') + parser.add_argument('-n', '--name', help='Name of the executable') + parser.add_argument('--onefile', action='store_true', help='Create a single executable file') + parser.add_argument('--windowed', action='store_true', help='Create windowed application (no console)') + parser.add_argument('--debug', action='store_true', help='Enable debug mode') + parser.add_argument('--exclude', action='append', help='Exclude modules') + parser.add_argument('--include', action='append', help='Include additional modules') + parser.add_argument('--add-data', action='append', help='Add data files (format: src:dest)') + + args = parser.parse_args() + + # Initialize logger + logger = Logger(debug=args.debug) + + # Validate input script + script_path = Path(args.script) + if not script_path.exists(): + logger.error(f"Script not found: {script_path}") + sys.exit(1) + + if not script_path.suffix == '.py': + logger.error("Input must be a Python script (.py)") + sys.exit(1) + + # Determine output name + if args.name: + output_name = args.name + elif args.output: + output_name = args.output + else: + output_name = script_path.stem + + # Initialize configuration + config = ConfigManager( + script_path=script_path, + output_name=output_name, + onefile=args.onefile, + windowed=args.windowed, + debug=args.debug, + exclude_modules=args.exclude or [], + include_modules=args.include or [], + add_data=args.add_data or [] + ) + + try: + logger.info(f"Converting Python {script_path} to executable...") + + # Step 1: Analyze dependencies + logger.info("Analyzing dependencies...") + analyzer = DependencyAnalyzer(config, logger) + dependencies = analyzer.analyze() + + # Step 2: Collect code and resources + logger.info("Collecting code and resources...") + collector = CodeCollector(config, logger) + collected_files = collector.collect(dependencies) + + # Step 3: Compile to bytecode + logger.info("Compiling to bytecode...") + compiler = BytecodeCompiler(config, logger) + bytecode_files = compiler.compile(collected_files) + + # Step 4: Build executable + logger.info("Building executable...") + builder = ExecutableBuilder(config, logger) + executable_path = builder.build(bytecode_files) + + logger.info(f"Executable created: {executable_path}") + + except Exception as e: + logger.error(f"Build failed: {e}") + if args.debug: + import traceback + traceback.print_exc() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/scripts/bellande_python_executable_test.sh b/scripts/bellande_python_executable_test.sh new file mode 100755 index 0000000..69a036a --- /dev/null +++ b/scripts/bellande_python_executable_test.sh @@ -0,0 +1,5 @@ +python3 main.py -script ../test_files/test.py \ + --output Test \ + -n Test_Name \ + --onefile \ + --windowed diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 0000000..48934a9 --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,2 @@ +python setup.py sdist +twine upload dist/* diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e69de29 diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..f6b8568 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +bellande_rust_executable.egg-info diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/analyzer.py b/src/analyzer.py new file mode 100644 index 0000000..f39b8ac --- /dev/null +++ b/src/analyzer.py @@ -0,0 +1,203 @@ +""" +Dependency analyzer for PyPack +Analyzes Python files to find all imports and dependencies +""" + +from header_imports import * + +class DependencyAnalyzer: + """Analyzes Python files to find dependencies""" + + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.analyzed_files = set() + self.dependencies = set() + self.import_graph = {} + + def analyze(self) -> Dict[str, Set[str]]: + """Analyze the main script and return all dependencies""" + self.logger.debug("Starting dependency analysis") + + # Start with the main script + self._analyze_file(self.config.script_path) + + # Add explicitly included modules + for module in self.config.include_modules: + self._add_module_dependency(module) + + # Remove excluded modules + for module in self.config.exclude_modules: + self.dependencies.discard(module) + + # Categorize dependencies + result = { + 'builtin': set(), + 'stdlib': set(), + 'third_party': set(), + 'local': set() + } + + for dep in self.dependencies: + if is_builtin_module(dep): + result['builtin'].add(dep) + elif is_stdlib_module(dep): + result['stdlib'].add(dep) + elif self._is_local_module(dep): + result['local'].add(dep) + else: + result['third_party'].add(dep) + + self.logger.debug(f"Found {len(self.dependencies)} dependencies") + self.logger.debug(f"Builtin: {len(result['builtin'])}") + self.logger.debug(f"Stdlib: {len(result['stdlib'])}") + self.logger.debug(f"Third-party: {len(result['third_party'])}") + self.logger.debug(f"Local: {len(result['local'])}") + + return result + + def _analyze_file(self, file_path: Path): + """Analyze a single Python file""" + if file_path in self.analyzed_files: + return + + self.analyzed_files.add(file_path) + self.logger.debug(f"Analyzing {file_path}") + + try: + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError: + try: + with open(file_path, 'r', encoding='latin-1') as f: + content = f.read() + except Exception as e: + self.logger.warning(f"Could not read {file_path}: {e}") + return + + try: + tree = ast.parse(content) + except SyntaxError as e: + self.logger.warning(f"Syntax error in {file_path}: {e}") + return + + # Find all imports + imports = self._extract_imports(tree) + + # Add to dependencies + for imp in imports: + self.dependencies.add(imp) + self._add_module_dependency(imp) + + # Find local imports and analyze them + for imp in imports: + if self._is_local_module(imp): + local_path = self._find_local_module_path(imp, file_path.parent) + if local_path: + self._analyze_file(local_path) + + def _extract_imports(self, tree: ast.AST) -> List[str]: + """Extract import statements from AST""" + imports = [] + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + imports.append(alias.name.split('.')[0]) + + elif isinstance(node, ast.ImportFrom): + if node.module: + imports.append(node.module.split('.')[0]) + else: + # Relative import + imports.append('.') + + return imports + + def _add_module_dependency(self, module_name: str): + """Add a module and its dependencies""" + if module_name in ['', '.']: + return + + try: + spec = importlib.util.find_spec(module_name) + if spec is None: + self.logger.warning(f"Module not found: {module_name}") + return + + # If it's a package, try to find submodules + if spec.submodule_search_locations: + self._find_package_modules(module_name, spec.submodule_search_locations) + + except ImportError as e: + self.logger.warning(f"Could not import {module_name}: {e}") + + def _find_package_modules(self, package_name: str, search_paths: List[str]): + """Find modules in a package""" + for search_path in search_paths: + path = Path(search_path) + if path.exists(): + for py_file in path.glob('*.py'): + if py_file.name != '__init__.py': + module_name = f"{package_name}.{py_file.stem}" + self.dependencies.add(module_name) + + def _is_local_module(self, module_name: str) -> bool: + """Check if a module is local to the project""" + if module_name in ['', '.']: + return False + + try: + spec = importlib.util.find_spec(module_name) + if spec is None or spec.origin is None: + return False + + # Check if the module is in the project directory + project_dir = self.config.script_path.parent + module_path = Path(spec.origin) + + try: + module_path.relative_to(project_dir) + return True + except ValueError: + return False + + except ImportError: + return False + + def _find_local_module_path(self, module_name: str, base_path: Path) -> Optional[Path]: + """Find the path to a local module""" + # Try direct .py file + py_file = base_path / f"{module_name}.py" + if py_file.exists(): + return py_file + + # Try package directory + package_dir = base_path / module_name + if package_dir.is_dir(): + init_file = package_dir / "__init__.py" + if init_file.exists(): + return init_file + + # Try searching in parent directories + parent = base_path.parent + if parent != base_path: # Not at root + return self._find_local_module_path(module_name, parent) + + return None + +class ImportVisitor(ast.NodeVisitor): + """AST visitor to find imports""" + + def __init__(self): + self.imports = [] + + def visit_Import(self, node): + for alias in node.names: + self.imports.append(alias.name) + self.generic_visit(node) + + def visit_ImportFrom(self, node): + if node.module: + self.imports.append(node.module) + self.generic_visit(node) diff --git a/src/builder.py b/src/builder.py new file mode 100644 index 0000000..8e2cfdf --- /dev/null +++ b/src/builder.py @@ -0,0 +1,278 @@ +""" +Executable builder for PyPack +Creates the final executable with embedded Python runtime +""" + +from header_imports import * + +class ExecutableBuilder: + """Builds the final executable""" + + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.platform_info = get_platform_info() + + def build(self, compiled_files: Dict[str, Path]) -> Path: + """Build the final executable""" + self.logger.debug("Starting executable build") + + # Create the bootstrap C code + bootstrap_c = self._create_bootstrap_code(compiled_files) + + # Write bootstrap code to temporary file + bootstrap_path = create_temp_file(bootstrap_c, '.c') + + try: + # Compile the executable + executable_path = self._compile_executable(bootstrap_path, compiled_files) + + # Make executable on Unix-like systems + if self.platform_info['system'] in ['linux', 'darwin']: + os.chmod(executable_path, 0o755) + + self.logger.debug(f"Built executable: {executable_path}") + return executable_path + + finally: + # Clean up temporary files + try: + os.unlink(bootstrap_path) + except: + pass + + def _create_bootstrap_code(self, compiled_files: Dict[str, Path]) -> str: + """Create the C bootstrap code""" + template = self._get_bootstrap_template() + + # Read main script bytecode + main_script_data = "" + if compiled_files.get('main_script'): + with open(compiled_files['main_script'], 'rb') as f: + bytecode = f.read() + main_script_data = self._bytes_to_c_array(bytecode) + + # Read archive data + archives_data = {} + for category in ['stdlib_modules', 'third_party_modules', 'local_modules', 'data_files']: + if compiled_files.get(category): + with open(compiled_files[category], 'rb') as f: + archive_data = f.read() + archives_data[category] = self._bytes_to_c_array(archive_data) + + # Fill in template + code = template.format( + main_script_data=main_script_data, + main_script_size=len(main_script_data.split(',')) if main_script_data else 0, + stdlib_data=archives_data.get('stdlib_modules', ''), + stdlib_size=len(archives_data.get('stdlib_modules', '').split(',')) if archives_data.get('stdlib_modules') else 0, + third_party_data=archives_data.get('third_party_modules', ''), + third_party_size=len(archives_data.get('third_party_modules', '').split(',')) if archives_data.get('third_party_modules') else 0, + local_data=archives_data.get('local_modules', ''), + local_size=len(archives_data.get('local_modules', '').split(',')) if archives_data.get('local_modules') else 0, + data_files_data=archives_data.get('data_files', ''), + data_files_size=len(archives_data.get('data_files', '').split(',')) if archives_data.get('data_files') else 0, + ) + + return code + + def _get_bootstrap_template(self) -> str: + """Get the C bootstrap template""" + return ''' +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#define PATH_SEP "\\\\" +#else +#include +#define PATH_SEP "/" +#endif + +#define PY_SSIZE_T_CLEAN +#include + +// Embedded data +static unsigned char main_script_data[] = {{{main_script_data}}}; +static size_t main_script_size = {main_script_size}; + +static unsigned char stdlib_data[] = {{{stdlib_data}}}; +static size_t stdlib_size = {stdlib_size}; + +static unsigned char third_party_data[] = {{{third_party_data}}}; +static size_t third_party_size = {third_party_size}; + +static unsigned char local_data[] = {{{local_data}}}; +static size_t local_size = {local_size}; + +static unsigned char data_files_data[] = {{{data_files_data}}}; +static size_t data_files_size = {data_files_size}; + +// Extract embedded data to temporary directory +static char* extract_data(unsigned char* data, size_t size, const char* filename) {{ + if (size == 0) return NULL; + + char* temp_dir = getenv("TMPDIR"); + if (!temp_dir) temp_dir = "/tmp"; + + char* filepath = malloc(strlen(temp_dir) + strlen(filename) + 20); + sprintf(filepath, "%s/pypacker_%d_%s", temp_dir, getpid(), filename); + + FILE* f = fopen(filepath, "wb"); + if (!f) {{ + free(filepath); + return NULL; + }} + + fwrite(data, 1, size, f); + fclose(f); + + return filepath; +}} + +// Custom import hook +static PyObject* custom_import(PyObject* self, PyObject* args) {{ + // This would implement custom import logic + // For now, use default import + return PyObject_CallMethod(PyImport_GetModuleDict(), "get", "s", "__import__"); +}} + +int main(int argc, char* argv[]) {{ + // Initialize Python + Py_Initialize(); + + if (!Py_IsInitialized()) {{ + fprintf(stderr, "Failed to initialize Python\\n"); + return 1; + }} + + // Set up sys.argv + PySys_SetArgv(argc, argv); + + // Extract and run main script + if (main_script_size > 0) {{ + // Load bytecode from embedded data + PyObject* code = PyMarshal_ReadObjectFromString((char*)main_script_data + 12, main_script_size - 12); + if (!code) {{ + PyErr_Print(); + Py_Finalize(); + return 1; + }} + + // Create main module + PyObject* main_module = PyImport_AddModule("__main__"); + if (!main_module) {{ + Py_DECREF(code); + Py_Finalize(); + return 1; + }} + + PyObject* main_dict = PyModule_GetDict(main_module); + + // Execute the code + PyObject* result = PyEval_EvalCode(code, main_dict, main_dict); + + Py_DECREF(code); + + if (!result) {{ + PyErr_Print(); + Py_Finalize(); + return 1; + }} + + Py_DECREF(result); + }} + + // Clean up + Py_Finalize(); + return 0; +}} +''' + + def _bytes_to_c_array(self, data: bytes) -> str: + """Convert bytes to C array format""" + if not data: + return "" + return ','.join(f'0x{b:02x}' for b in data) + + def _compile_executable(self, bootstrap_path: str, compiled_files: Dict[str, Path]) -> Path: + """Compile the C bootstrap into an executable""" + output_path = self.config.get_output_path(self.config.output_name) + + if self.platform_info['system'] == 'windows': + output_path = output_path.with_suffix('.exe') + + # Find Python includes and libraries + python_includes = self._get_python_includes() + python_libs = self._get_python_libs() + + # Build compiler command + if self.platform_info['system'] == 'windows': + # Windows with MSVC + cmd = [ + 'cl', + '/nologo', + f'/I{python_includes}', + bootstrap_path, + f'/Fe{output_path}', + f'/link', f'/LIBPATH:{python_libs}', + 'python3.lib' + ] + else: + # Unix-like systems with GCC + cmd = [ + 'gcc', + '-o', str(output_path), + f'-I{python_includes}', + bootstrap_path, + f'-L{python_libs}', + '-lpython3.11', # Adjust version as needed + '-ldl', '-lm' + ] + + self.logger.debug(f"Compiler command: {' '.join(cmd)}") + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + self.logger.debug("Compilation successful") + return output_path + except subprocess.CalledProcessError as e: + self.logger.error(f"Compilation failed: {e}") + self.logger.error(f"Stdout: {e.stdout}") + self.logger.error(f"Stderr: {e.stderr}") + raise + + def _get_python_includes(self) -> str: + """Get Python include directory""" + import sysconfig + return sysconfig.get_path('include') + + def _get_python_libs(self) -> str: + """Get Python library directory""" + import sysconfig + + if self.platform_info['system'] == 'windows': + return sysconfig.get_path('stdlib') + else: + # For Unix-like systems + return sysconfig.get_config_var('LIBDIR') or '/usr/lib' + + def _find_compiler(self) -> Optional[str]: + """Find a suitable C compiler""" + compilers = [] + + if self.platform_info['system'] == 'windows': + compilers = ['cl', 'gcc', 'clang'] + else: + compilers = ['gcc', 'clang', 'cc'] + + for compiler in compilers: + if shutil.which(compiler): + return compiler + + return None diff --git a/src/collector.py b/src/collector.py new file mode 100644 index 0000000..e9fc17c --- /dev/null +++ b/src/collector.py @@ -0,0 +1,233 @@ +""" +Code collector for PyPack +Gathers all Python source files and modules needed for the executable +""" + +from header_imports import * + +class CodeCollector: + """Collects all necessary Python files and resources""" + + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.collected_files = {} + self.python_paths = get_python_paths() + + def collect(self, dependencies: Dict[str, Set[str]]) -> Dict[str, List[Path]]: + """Collect all necessary files""" + self.logger.debug("Starting code collection") + + result = { + 'main_script': [], + 'stdlib_modules': [], + 'third_party_modules': [], + 'local_modules': [], + 'data_files': [], + 'python_dll': None + } + + # Collect main script + result['main_script'] = [self.config.script_path] + + # Collect standard library modules + for module in dependencies['stdlib']: + files = self._collect_stdlib_module(module) + result['stdlib_modules'].extend(files) + + # Collect third-party modules + for module in dependencies['third_party']: + files = self._collect_third_party_module(module) + result['third_party_modules'].extend(files) + + # Collect local modules + for module in dependencies['local']: + files = self._collect_local_module(module) + result['local_modules'].extend(files) + + # Collect additional data files + for data_spec in self.config.add_data: + files = self._collect_data_files(data_spec) + result['data_files'].extend(files) + + # Find Python DLL/SO + from utils import find_python_dll + python_dll = find_python_dll() + if python_dll: + result['python_dll'] = python_dll + else: + self.logger.warning("Could not find Python DLL/SO - executable may not work") + + self.logger.debug(f"Collected {sum(len(v) for v in result.values() if isinstance(v, list))} files") + + return result + + def _collect_stdlib_module(self, module_name: str) -> List[Path]: + """Collect standard library module files""" + files = [] + + try: + spec = importlib.util.find_spec(module_name) + if spec is None: + self.logger.warning(f"Standard library module not found: {module_name}") + return files + + if spec.origin: + # Single file module + files.append(Path(spec.origin)) + elif spec.submodule_search_locations: + # Package + for location in spec.submodule_search_locations: + path = Path(location) + if path.exists(): + files.extend(self._collect_package_files(path)) + + except ImportError as e: + self.logger.warning(f"Could not collect stdlib module {module_name}: {e}") + + return files + + def _collect_third_party_module(self, module_name: str) -> List[Path]: + """Collect third-party module files""" + files = [] + + try: + spec = importlib.util.find_spec(module_name) + if spec is None: + self.logger.warning(f"Third-party module not found: {module_name}") + return files + + if spec.origin: + # Single file module + files.append(Path(spec.origin)) + + # Also collect any related files (e.g., .so files) + module_dir = Path(spec.origin).parent + module_stem = Path(spec.origin).stem + + # Look for compiled extensions + for ext in ['.so', '.pyd', '.dll']: + ext_file = module_dir / f"{module_stem}{ext}" + if ext_file.exists(): + files.append(ext_file) + + elif spec.submodule_search_locations: + # Package + for location in spec.submodule_search_locations: + path = Path(location) + if path.exists(): + files.extend(self._collect_package_files(path)) + + except ImportError as e: + self.logger.warning(f"Could not collect third-party module {module_name}: {e}") + + return files + + def _collect_local_module(self, module_name: str) -> List[Path]: + """Collect local module files""" + files = [] + + try: + spec = importlib.util.find_spec(module_name) + if spec is None: + self.logger.warning(f"Local module not found: {module_name}") + return files + + if spec.origin: + files.append(Path(spec.origin)) + elif spec.submodule_search_locations: + for location in spec.submodule_search_locations: + path = Path(location) + if path.exists(): + files.extend(self._collect_package_files(path)) + + except ImportError as e: + self.logger.warning(f"Could not collect local module {module_name}: {e}") + + return files + + def _collect_package_files(self, package_path: Path) -> List[Path]: + """Collect all files in a package directory""" + files = [] + + for item in package_path.rglob('*'): + if item.is_file(): + # Include Python files + if item.suffix in ['.py', '.pyx']: + files.append(item) + # Include compiled extensions + elif item.suffix in ['.so', '.pyd', '.dll']: + files.append(item) + # Include data files in packages + elif item.suffix in ['.txt', '.json', '.xml', '.yaml', '.yml', '.cfg', '.ini']: + files.append(item) + + return files + + def _collect_data_files(self, data_spec: str) -> List[Path]: + """Collect data files specified by user""" + files = [] + + if ':' in data_spec: + src, dest = data_spec.split(':', 1) + else: + src = data_spec + dest = None + + src_path = Path(src) + + if src_path.is_file(): + files.append(src_path) + elif src_path.is_dir(): + files.extend(src_path.rglob('*')) + else: + self.logger.warning(f"Data file not found: {src}") + + return files + + def copy_to_work_dir(self, collected_files: Dict[str, List[Path]]) -> Dict[str, List[Path]]: + """Copy collected files to work directory""" + self.logger.debug("Copying files to work directory") + + result = {} + + for category, files in collected_files.items(): + if category == 'python_dll': + # Special handling for Python DLL + if files: + dll_dest = self.config.get_work_path(files.name) + shutil.copy2(files, dll_dest) + result[category] = dll_dest + continue + + result[category] = [] + + for file_path in files: + if not isinstance(file_path, Path): + continue + + # Determine destination path + if category == 'main_script': + dest_path = self.config.get_work_path(file_path.name) + elif category == 'local_modules': + # Preserve relative structure for local modules + try: + rel_path = file_path.relative_to(self.config.script_path.parent) + dest_path = self.config.get_work_path('local', rel_path) + except ValueError: + dest_path = self.config.get_work_path('local', file_path.name) + else: + # For stdlib and third-party, preserve the package structure + dest_path = self.config.get_work_path(category, file_path.name) + + # Create destination directory + dest_path.parent.mkdir(parents=True, exist_ok=True) + + # Copy file + try: + shutil.copy2(file_path, dest_path) + result[category].append(dest_path) + except Exception as e: + self.logger.warning(f"Could not copy {file_path}: {e}") + + return result diff --git a/src/compiler.py b/src/compiler.py new file mode 100644 index 0000000..4624482 --- /dev/null +++ b/src/compiler.py @@ -0,0 +1,122 @@ +""" +Bytecode compiler for PyPack +Compiles Python source files to bytecode for distribution +""" + +from header_imports import * + +class BytecodeCompiler: + """Compiles Python source files to bytecode""" + + def __init__(self, config, logger): + self.config = config + self.logger = logger + + def compile(self, collected_files: Dict[str, List[Path]]) -> Dict[str, Path]: + """Compile all Python files to bytecode and create archives""" + self.logger.debug("Starting bytecode compilation") + + result = {} + + # Compile main script + if collected_files['main_script']: + main_script = collected_files['main_script'][0] + compiled_main = self._compile_single_file(main_script) + result['main_script'] = compiled_main + + # Create archives for different categories + for category in ['stdlib_modules', 'third_party_modules', 'local_modules']: + if collected_files[category]: + archive_path = self._create_module_archive(category, collected_files[category]) + result[category] = archive_path + + # Handle data files + if collected_files['data_files']: + data_archive = self._create_data_archive(collected_files['data_files']) + result['data_files'] = data_archive + + # Copy Python DLL + if collected_files.get('python_dll'): + result['python_dll'] = collected_files['python_dll'] + + return result + + def _compile_single_file(self, source_path: Path) -> Path: + """Compile a single Python file to bytecode""" + output_path = self.config.get_work_path(f"{source_path.stem}.pyc") + + try: + py_compile.compile(source_path, output_path, doraise=True) + self.logger.debug(f"Compiled {source_path} to {output_path}") + return output_path + except py_compile.PyCompileError as e: + self.logger.error(f"Failed to compile {source_path}: {e}") + raise + + def _create_module_archive(self, category: str, files: List[Path]) -> Path: + """Create a ZIP archive containing compiled modules""" + archive_path = self.config.get_work_path(f"{category}.zip") + + with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for file_path in files: + if file_path.suffix == '.py': + # Compile Python file + try: + compiled_path = self._compile_python_to_bytecode(file_path) + arcname = file_path.stem + '.pyc' + zipf.write(compiled_path, arcname) + # Clean up temporary compiled file + compiled_path.unlink() + except Exception as e: + self.logger.warning(f"Could not compile {file_path}: {e}") + # Fall back to source + arcname = file_path.name + zipf.write(file_path, arcname) + else: + # Copy non-Python files as-is + arcname = file_path.name + zipf.write(file_path, arcname) + + self.logger.debug(f"Created module archive: {archive_path}") + return archive_path + + def _create_data_archive(self, files: List[Path]) -> Path: + """Create a ZIP archive containing data files""" + archive_path = self.config.get_work_path("data_files.zip") + + with zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + for file_path in files: + if file_path.is_file(): + # Preserve directory structure + arcname = file_path.name + zipf.write(file_path, arcname) + + self.logger.debug(f"Created data archive: {archive_path}") + return archive_path + + def _compile_python_to_bytecode(self, source_path: Path) -> Path: + """Compile Python source to bytecode""" + output_path = source_path.with_suffix('.pyc') + + with open(source_path, 'r', encoding='utf-8') as f: + source_code = f.read() + + try: + # Compile to code object + code_obj = compile(source_code, str(source_path), 'exec') + + # Write bytecode file + with open(output_path, 'wb') as f: + # Write magic number and timestamp + f.write(py_compile.MAGIC) + f.write(b'\x00\x00\x00\x00') # timestamp + f.write(b'\x00\x00\x00\x00') # size + + # Write marshalled code + marshal.dump(code_obj, f) + + return output_path + + except SyntaxError as e: + self.logger.error(f"Syntax error in {source_path}: {e}") + raise diff --git a/src/utilities.py b/src/utilities.py new file mode 100644 index 0000000..973c9ba --- /dev/null +++ b/src/utilities.py @@ -0,0 +1,199 @@ +""" +Utility classes and functions for PyPack +""" + +from header_imports import * + +class Logger: + """Simple logging utility""" + + def __init__(self, debug=False): + self.debug_mode = debug + + def info(self, message): + print(f"[INFO] {message}") + + def warning(self, message): + print(f"[WARNING] {message}") + + def error(self, message): + print(f"[ERROR] {message}") + + def debug(self, message): + if self.debug_mode: + print(f"[DEBUG] {message}") + +@dataclass +class ConfigManager: + """Configuration manager for build settings""" + script_path: Path + output_name: str + onefile: bool = True + windowed: bool = False + debug: bool = False + exclude_modules: List[str] = None + include_modules: List[str] = None + add_data: List[str] = None + + def __post_init__(self): + if self.exclude_modules is None: + self.exclude_modules = [] + if self.include_modules is None: + self.include_modules = [] + if self.add_data is None: + self.add_data = [] + + # Create work directory + self.work_dir = Path(f"build_{self.output_name}_{int(time.time())}") + self.work_dir.mkdir(exist_ok=True) + + # Output directory + self.output_dir = Path("dist") + self.output_dir.mkdir(exist_ok=True) + + def get_work_path(self, *args): + """Get path relative to work directory""" + return self.work_dir / Path(*args) + + def get_output_path(self, *args): + """Get path relative to output directory""" + return self.output_dir / Path(*args) + +def get_python_paths(): + """Get Python installation paths""" + import sysconfig + + paths = { + 'executable': sys.executable, + 'stdlib': sysconfig.get_path('stdlib'), + 'platstdlib': sysconfig.get_path('platstdlib'), + 'purelib': sysconfig.get_path('purelib'), + 'platlib': sysconfig.get_path('platlib'), + 'include': sysconfig.get_path('include'), + 'data': sysconfig.get_path('data'), + } + + return paths + +def get_platform_info(): + """Get platform-specific information""" + import platform + + return { + 'system': platform.system().lower(), + 'machine': platform.machine().lower(), + 'architecture': platform.architecture()[0], + 'python_version': platform.python_version(), + } + +def find_python_dll(): + """Find Python DLL/SO file""" + import sysconfig + + platform_info = get_platform_info() + + if platform_info['system'] == 'windows': + # Windows: look for pythonXX.dll + version = sys.version_info + dll_name = f"python{version.major}{version.minor}.dll" + + # Check common locations + locations = [ + Path(sys.executable).parent / dll_name, + Path(sys.executable).parent / "DLLs" / dll_name, + Path(sysconfig.get_path('stdlib')) / dll_name, + ] + + for location in locations: + if location.exists(): + return location + + elif platform_info['system'] == 'linux': + # Linux: look for libpythonX.Y.so + version = sys.version_info + so_name = f"libpython{version.major}.{version.minor}.so" + + # Check common locations + locations = [ + Path(f"/usr/lib/x86_64-linux-gnu/{so_name}"), + Path(f"/usr/lib/{so_name}"), + Path(f"/usr/local/lib/{so_name}"), + Path(sysconfig.get_path('stdlib')) / ".." / "lib" / so_name, + ] + + for location in locations: + if location.exists(): + return location + + # Try to find any libpython*.so + import glob + for pattern in ["/usr/lib/*/libpython*.so*", "/usr/local/lib/libpython*.so*"]: + matches = glob.glob(pattern) + if matches: + return Path(matches[0]) + + elif platform_info['system'] == 'darwin': + # macOS: look for libpythonX.Y.dylib + version = sys.version_info + dylib_name = f"libpython{version.major}.{version.minor}.dylib" + + locations = [ + Path(f"/usr/local/lib/{dylib_name}"), + Path(f"/opt/homebrew/lib/{dylib_name}"), + Path(sysconfig.get_path('stdlib')) / ".." / "lib" / dylib_name, + ] + + for location in locations: + if location.exists(): + return location + + return None + +def is_builtin_module(module_name): + """Check if a module is a built-in module""" + return module_name in sys.builtin_module_names + +def is_stdlib_module(module_name): + """Check if a module is part of the standard library""" + import importlib.util + + if is_builtin_module(module_name): + return True + + try: + spec = importlib.util.find_spec(module_name) + if spec is None: + return False + + if spec.origin is None: + return True # namespace package, likely stdlib + + # Check if the module is in the standard library path + stdlib_path = Path(get_python_paths()['stdlib']) + platstdlib_path = Path(get_python_paths()['platstdlib']) + + module_path = Path(spec.origin) + + try: + module_path.relative_to(stdlib_path) + return True + except ValueError: + pass + + try: + module_path.relative_to(platstdlib_path) + return True + except ValueError: + pass + + return False + except ImportError: + return False + +def create_temp_file(content, suffix=".c"): + """Create a temporary file with content""" + import tempfile + + with tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False) as f: + f.write(content) + return f.name diff --git a/test_files/test.py b/test_files/test.py new file mode 100644 index 0000000..ad35e5a --- /dev/null +++ b/test_files/test.py @@ -0,0 +1 @@ +print("Hello World")