diff --git a/Ghidra/Features/PyGhidra/.pydevproject b/Ghidra/Features/PyGhidra/.pydevproject index 72a29a46c5..bca7139ae3 100644 --- a/Ghidra/Features/PyGhidra/.pydevproject +++ b/Ghidra/Features/PyGhidra/.pydevproject @@ -1,12 +1,17 @@ - + + Default - + + python interpreter + /${PROJECT_DIR_NAME}/ghidra_scripts /${PROJECT_DIR_NAME}/src/main/py/src/pyghidra + /${PROJECT_DIR_NAME}/support + diff --git a/Ghidra/Features/PyGhidra/support/pyghidra_launcher.py b/Ghidra/Features/PyGhidra/support/pyghidra_launcher.py index 5752b71f91..c618ef5606 100644 --- a/Ghidra/Features/PyGhidra/support/pyghidra_launcher.py +++ b/Ghidra/Features/PyGhidra/support/pyghidra_launcher.py @@ -19,10 +19,9 @@ import os import sys import subprocess import sysconfig -import venv from pathlib import Path -from typing import List, Dict -from sys import stderr, version +from itertools import chain +from typing import List, Dict, Tuple def get_application_properties(install_dir: Path) -> Dict[str, str]: app_properties_path: Path = install_dir / 'Ghidra' / 'application.properties' @@ -37,12 +36,14 @@ def get_application_properties(install_dir: Path) -> Dict[str, str]: props[key] = value return props -def get_user_settings_dir(install_dir: Path) -> Path: +def get_user_settings_dir(install_dir: Path, dev: bool) -> Path: props: Dict[str, str] = get_application_properties(install_dir) app_name: str = props['application.name'].replace(' ', '').lower() app_version: str = props['application.version'] app_release_name: str = props['application.release.name'] versioned_name: str = f'{app_name}_{app_version}_{app_release_name}' + if dev: + versioned_name += f'_location_{install_dir.parent.name}' xdg_config_home: str = os.environ.get('XDG_CONFIG_HOME') if xdg_config_home: return Path(xdg_config_home) / app_name / versioned_name @@ -51,54 +52,97 @@ def get_user_settings_dir(install_dir: Path) -> Path: if platform.system() == 'Darwin': return Path.home() / 'Library' / app_name / versioned_name return Path.home() / '.config' / app_name / versioned_name + +def find_supported_python_exe(install_dir: Path, dev: bool) -> List[str]: + python_cmds = [] + saved_python_cmd = get_saved_python_cmd(install_dir, dev) + if saved_python_cmd is not None: + python_cmds.append(saved_python_cmd) + print("Last used Python executable: " + str(saved_python_cmd)) + + props: Dict[str, str] = get_application_properties(install_dir) + prop: str = 'application.python.supported' + supported: List[str] = [s.strip() for s in props.get(prop, '').split(',')] + if '' in supported: + raise ValueError(f'Invalid "{prop}" value in application.properties file') + + python_cmds += list(chain.from_iterable([[f'python{s}'], ['py', f'-{s}']] for s in supported)) + python_cmds += [['python3'], ['python'], ['py']] + + for cmd in python_cmds: + try: + result = subprocess.run(cmd + ['-c', 'import sys; print("{0}.{1}".format(*sys.version_info))'], capture_output=True, text=True) + version = result.stdout.strip() + if result.returncode == 0 and version in supported: + return cmd + except FileNotFoundError: + pass + + return None def in_venv() -> bool: return sys.prefix != sys.base_prefix def is_externally_managed() -> bool: - marker: Path = Path(sysconfig.get_path("stdlib", sysconfig.get_default_scheme())) / 'EXTERNALLY-MANAGED' + marker: Path = Path(sysconfig.get_path('stdlib', sysconfig.get_default_scheme())) / 'EXTERNALLY-MANAGED' return marker.is_file() -def get_venv_exe(venv_dir: Path) -> str: +def get_venv_exe(venv_dir: Path) -> List[str]: win_python_cmd: str = str(venv_dir / 'Scripts' / 'python.exe') linux_python_cmd: str = str(venv_dir / 'bin' / 'python3') - return win_python_cmd if platform.system() == 'Windows' else linux_python_cmd + return [win_python_cmd] if platform.system() == 'Windows' else [linux_python_cmd] -def get_ghidra_venv(install_dir: Path) -> Path: - user_settings_dir: Path = get_user_settings_dir(install_dir) - venv_dir: Path = user_settings_dir / 'venv' - return venv_dir +def get_ghidra_venv(install_dir: Path, dev: bool) -> Path: + return (install_dir / 'build' if dev else get_user_settings_dir(install_dir, dev)) / 'venv' -def create_ghidra_venv(venv_dir: Path) -> None: - print(f'Creating Ghidra virtual environemnt at {venv_dir}...') - venv.create(venv_dir, with_pip=True) +def create_ghidra_venv(python_cmd: List[str], venv_dir: Path) -> None: + print(f'Creating Ghidra virtual environment at {venv_dir}...') + subprocess.run(python_cmd + ['-m', 'venv', venv_dir.absolute()]) -def version_tuple(v): - filled = [] - for point in v.split("."): - filled.append(point.zfill(8)) - return tuple(filled) +def version_tuple(v: str) -> Tuple[str, ...]: + filled = [] + for point in v.split("."): + filled.append(point.zfill(8)) + return tuple(filled) -def get_package_version(python_cmd: str, package: str) -> str: +def get_package_version(python_cmd: List[str], package: str) -> str: version = None - result = subprocess.Popen([python_cmd, '-m', 'pip', 'show', package], stdout=subprocess.PIPE, text=True) - for line in result.stdout.readlines(): + result = subprocess.run(python_cmd + ['-m', 'pip', 'show', package], capture_output=True, text=True) + for line in result.stdout.splitlines(): line = line.strip() print(line) key, value = line.split(':', 1) if key == 'Version': version = value.strip() return version + +def get_saved_python_cmd(install_dir: Path, dev: bool) -> List[str]: + user_settings_dir: Path = get_user_settings_dir(install_dir, dev) + save_file: Path = user_settings_dir / 'python_command.save' + if not save_file.is_file(): + return None + ret = [] + with open(save_file, 'r') as f: + for line in f: + ret.append(line.strip()) + return ret + +def save_python_cmd(install_dir: Path, python_cmd: List[str], dev: bool) -> None: + user_settings_dir: Path = get_user_settings_dir(install_dir, dev) + save_file: Path = user_settings_dir / 'python_command.save' + with open(save_file, 'w') as f: + f.write('\n'.join(python_cmd) + '\n') -def install(install_dir: Path, python_cmd: str, pip_args: List[str], offer_venv: bool) -> bool: +def install(install_dir: Path, python_cmd: List[str], pip_args: List[str], offer_venv: bool) -> List[str]: install_choice: str = input('Do you wish to install PyGhidra (y/n)? ') if install_choice.lower() in ('y', 'yes'): if offer_venv: ghidra_venv_choice: str = input('Install into new Ghidra virtual environment (y/n)? ') if ghidra_venv_choice.lower() in ('y', 'yes'): - venv_dir = get_ghidra_venv(install_dir) - create_ghidra_venv(venv_dir) + venv_dir = get_ghidra_venv(install_dir, False) + create_ghidra_venv(python_cmd, venv_dir) python_cmd = get_venv_exe(venv_dir) + print(f'Switching to Ghidra virtual environment: {venv_dir}') elif ghidra_venv_choice.lower() in ('n', 'no'): system_venv_choice: str = input('Install into system environment (y/n)? ') if not system_venv_choice.lower() in ('y', 'yes'): @@ -107,24 +151,24 @@ def install(install_dir: Path, python_cmd: str, pip_args: List[str], offer_venv: else: print('Please answer yes or no.') return None - subprocess.check_call([python_cmd] + pip_args) + subprocess.check_call(python_cmd + pip_args) return python_cmd elif not install_choice.lower() in ('n', 'no'): print('Please answer yes or no.') return None -def upgrade(python_cmd: str, pip_args: List[str], dist_dir: Path, current_pyghidra_version: str) -> bool: +def upgrade(python_cmd: List[str], pip_args: List[str], dist_dir: Path, current_pyghidra_version: str) -> bool: included_pyghidra: Path = next(dist_dir.glob('pyghidra-*.whl'), None) if included_pyghidra is None: - print('Warning: included pyghidra wheel was not found', file=sys.stderr) - return + print('Warning: included pyghidra wheel was not found', file=sys.stderr) + return included_version = included_pyghidra.name.split('-')[1] current_version = current_pyghidra_version if version_tuple(included_version) > version_tuple(current_version): choice: str = input(f'Do you wish to upgrade PyGhidra {current_version} to {included_version} (y/n)? ') if choice.lower() in ('y', 'yes'): pip_args.append('-U') - subprocess.check_call([python_cmd] + pip_args) + subprocess.check_call(python_cmd + pip_args) return True else: print('Skipping upgrade') @@ -140,24 +184,30 @@ def main() -> None: args, remaining = parser.parse_known_args() # Setup variables - python_cmd: str = sys.executable - install_dir: Path = Path(args.install_dir) + install_dir: Path = Path(os.path.normpath(args.install_dir)) pyghidra_dir: Path = install_dir / 'Ghidra' / 'Features' / 'PyGhidra' dist_dir: Path = pyghidra_dir / 'pypkg' / 'dist' - dev_venv_dir = install_dir / 'build' / 'venv' - release_venv_dir = get_ghidra_venv(install_dir) - + venv_dir = get_ghidra_venv(install_dir, args.dev) + python_cmd: List[str] = find_supported_python_exe(install_dir, args.dev) + + if python_cmd is not None: + print(f'Using Python command: "{" ".join(python_cmd)}"') + else: + print('Supported version of Python not found. Check application.properties file.') + sys.exit(1) + # If headless, force console mode if args.headless: args.console = True if args.dev: # If in dev mode, launch PyGhidra from the source tree using the development virtual environment - if not dev_venv_dir.is_dir(): + if not venv_dir.is_dir(): print('Virtual environment not found!') print('Run "gradle prepdev" and try again.') sys.exit(1) - python_cmd = get_venv_exe(dev_venv_dir) + python_cmd = get_venv_exe(venv_dir) + print(f'Switchiing to Ghidra virtual environment: {venv_dir}') else: # If in release mode, offer to install or upgrade PyGhidra before launching from user-controlled environment pip_args: List[str] = ['-m', 'pip', 'install', '--no-index', '-f', str(dist_dir), 'pyghidra'] @@ -170,14 +220,15 @@ def main() -> None: if in_venv(): # If we are already in a virtual environment, assume that's where the user wants to be print(f'Using active virtual environment: {sys.prefix}') - elif os.path.isdir(release_venv_dir): + elif os.path.isdir(venv_dir): # If the Ghidra user settings venv exists, use that - python_cmd = get_venv_exe(release_venv_dir) - print(f'Using Ghidra virtual environment: {release_venv_dir}') + python_cmd = get_venv_exe(venv_dir) + print(f'Switching to Ghidra virtual environment: {venv_dir}') elif is_externally_managed(): print('Externally managed environment detected!') - create_ghidra_venv(release_venv_dir) - python_cmd = get_venv_exe(release_venv_dir) + create_ghidra_venv(python_cmd, venv_dir) + python_cmd = get_venv_exe(venv_dir) + print(f'Switching to Ghidra virtual environment: {venv_dir}') else: offer_venv = True @@ -192,7 +243,8 @@ def main() -> None: upgrade(python_cmd, pip_args, dist_dir, current_pyghidra_version) # Launch PyGhidra - py_args: List[str] = [python_cmd, '-m', 'pyghidra.ghidra_launch', '--install-dir', str(install_dir)] + save_python_cmd(install_dir, python_cmd, args.dev) + py_args: List[str] = python_cmd + ['-m', 'pyghidra.ghidra_launch', '--install-dir', str(install_dir)] if args.headless: py_args += ['ghidra.app.util.headless.AnalyzeHeadless'] else: diff --git a/Ghidra/Framework/Utility/src/main/java/ghidra/framework/ApplicationProperties.java b/Ghidra/Framework/Utility/src/main/java/ghidra/framework/ApplicationProperties.java index 0703b3ae44..998f961bb3 100644 --- a/Ghidra/Framework/Utility/src/main/java/ghidra/framework/ApplicationProperties.java +++ b/Ghidra/Framework/Utility/src/main/java/ghidra/framework/ApplicationProperties.java @@ -95,6 +95,12 @@ public class ApplicationProperties extends Properties { */ public static final String APPLICATION_JAVA_COMPILER_PROPERTY = "application.java.compiler"; + /** + * A comma-delimted priority-ordred list of versions of Python supported by the application. + */ + public static final String APPLICATION_PYTHON_SUPPORTED_PROPERTY = + "application.python.supported"; + /** * The date the application was built on, in a long format. * For example, "2018-Jan-11 1346 EST". diff --git a/Ghidra/application.properties b/Ghidra/application.properties index 18c5b61120..169b0191b5 100644 --- a/Ghidra/application.properties +++ b/Ghidra/application.properties @@ -7,3 +7,4 @@ application.gradle.max= application.java.min=21 application.java.max= application.java.compiler=21 +application.python.supported=3.12, 3.11, 3.10, 3.9 diff --git a/build.gradle b/build.gradle index 26d2333986..65d028f0c4 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,6 @@ if ("32".equals(System.getProperty("sun.arch.data.model"))) { /*************************************************************************************** * Identify supported Python command ***************************************************************************************/ -project.ext.SUPPORTED_PY_VERSIONS = ['3.12', '3.11', '3.10', '3.9'] project.ext.PYTHON3 = findPython3(true) project.ext.PYTHON_DEPS = new HashSet() @@ -219,12 +218,21 @@ def checkPip(List pyCmd, boolean shouldPrint) { } def findPython3(boolean shouldPrint) { - def pyCmds = SUPPORTED_PY_VERSIONS.collectMany { [["python$it"], ["py", "-$it"]] } - pyCmds += [['py'], ['python3'], ['python']] + def supportedVersions = "${PYTHON_SUPPORTED}".split(",").collect { + try { + GradleVersion.version(it.trim()).getVersion() // use GradleVersion to validate version format + } + catch (IllegalArgumentException e) { + throw new GradleException("Invalid supported Python version list specified in application.properties.\n" + e.message); + } + } + + def pyCmds = supportedVersions.collectMany { [["python$it"], ["py", "-$it"]] } + pyCmds += [['python3'], ['python'], ['py']] for (pyCmd in pyCmds) { def pyVer = checkPythonVersion(pyCmd) def pyExe = getPythonExecutable(pyCmd) - if (pyVer in SUPPORTED_PY_VERSIONS) { + if (pyVer in supportedVersions) { if (shouldPrint) { println("Python3 command: ${pyCmd} (${pyVer}, ${pyExe})") } @@ -234,7 +242,7 @@ def findPython3(boolean shouldPrint) { } if (shouldPrint) { - println("Warning: Supported Python ${SUPPORTED_PY_VERSIONS} not found (required for build)") + println("Warning: Supported Python [${PYTHON_SUPPORTED}] not found (required for build)") } // Don't fail until task execution. Just retun null, which can be gracefully handled later. diff --git a/gradle/hasPythonPackage.gradle b/gradle/hasPythonPackage.gradle index bb260eef05..d98f29c404 100644 --- a/gradle/hasPythonPackage.gradle +++ b/gradle/hasPythonPackage.gradle @@ -49,7 +49,7 @@ task buildPyPackage { doLast { if (rootProject.PYTHON3 == null) { - throw new GradleException("A supported version of Python ${SUPPORTED_PY_VERSIONS} was not found!") + throw new GradleException("A supported version of Python [${PYTHON_SUPPORTED}] was not found!") } File setuptools = project(":Debugger-rmi-trace").findPyDep(".") diff --git a/gradle/root/distribution.gradle b/gradle/root/distribution.gradle index 44da14c4b7..06f35ed853 100644 --- a/gradle/root/distribution.gradle +++ b/gradle/root/distribution.gradle @@ -271,7 +271,7 @@ task createGhidraStubsWheel { File setuptools = project(":Debugger-rmi-trace").findPyDep(".") if (PYTHON3 == null) { - throw new GradleException("A supported version of Python ${SUPPORTED_PY_VERSIONS} was not found!") + throw new GradleException("A supported version of Python [${PYTHON_SUPPORTED}] was not found!") } exec { diff --git a/gradle/root/venv.gradle b/gradle/root/venv.gradle index aa4cefad47..4178c83386 100644 --- a/gradle/root/venv.gradle +++ b/gradle/root/venv.gradle @@ -27,7 +27,7 @@ task createPythonVirtualEnvironment(type: Exec) { doFirst { if (rootProject.PYTHON3 == null) { - throw new GradleException("A supported version of Python ${SUPPORTED_PY_VERSIONS} was not found!") + throw new GradleException("A supported version of Python [${PYTHON_SUPPORTED}] was not found!") } commandLine rootProject.PYTHON3 diff --git a/gradle/support/loadApplicationProperties.gradle b/gradle/support/loadApplicationProperties.gradle index b20831c428..d502d24a79 100644 --- a/gradle/support/loadApplicationProperties.gradle +++ b/gradle/support/loadApplicationProperties.gradle @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,6 +28,7 @@ file("Ghidra/application.properties").withReader { reader -> project.ext.JAVA_COMPILER = ghidraProps.getProperty('application.java.compiler') project.ext.GRADLE_MIN = ghidraProps.getProperty('application.gradle.min') project.ext.GRADLE_MAX = ghidraProps.getProperty('application.gradle.max') + project.ext.PYTHON_SUPPORTED = ghidraProps.getProperty('application.python.supported') project.ext.DISTRO_PREFIX = "ghidra_${version}_${RELEASE_NAME}" // Build dates may or may not be already present in the application.properties file.