Python3 support

This commit is contained in:
DC3-TSD
2024-09-09 09:58:05 -04:00
parent d7c1f65f43
commit 92d0f1dacf
101 changed files with 11413 additions and 13 deletions
@@ -28,4 +28,5 @@ dependencies {
}
rootProject.createJsondocs.dependsOn jar
rootProject.createPythonTypeStubs.dependsOn jar
@@ -0,0 +1,156 @@
package ghidra.doclets.typestubs;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.stream.Collectors;
import javax.lang.model.element.Element;
import javax.lang.model.util.Elements;
import javax.tools.Diagnostic;
import com.sun.source.doctree.AttributeTree;
import com.sun.source.doctree.DocCommentTree;
import com.sun.source.doctree.DocTree;
import com.sun.source.util.DocTreePath;
import com.sun.source.util.DocTrees;
import com.sun.source.util.TreePath;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
/**
* Base class for recursively converting documentation
*/
abstract class DocConverter {
static final int INDENT_WIDTH = 4;
private final DocletEnvironment env;
private final Reporter log;
/**
* Creates a new {@link DocConverter}
*
* @param env the doclet environment
* @param log the log
*/
DocConverter(DocletEnvironment env, Reporter log) {
this.env = env;
this.log = log;
}
/**
* Converts the provided Javadoc tag
*
* @param el the current element
* @param tag the Javadoc tag
* @return the converted tag
*/
abstract String convertTag(Element el, DocTree tag, ListIterator<? extends DocTree> it);
/**
* Converts the provided doc tree
*
* @param el the current element
* @param tree the doc tree
* @return the converted doc tree
*/
public String convertTree(Element el, List<? extends DocTree> tree) {
StringBuilder builder = new StringBuilder();
ListIterator<? extends DocTree> it = tree.listIterator();
while (it.hasNext()) {
builder.append(convertTag(el, it.next(), it));
}
return builder.toString();
}
/**
* Logs a warning with the provided message
*
* @param el the current element
* @param tag the current tag
* @param message the message
*/
final void logWarning(Element el, DocTree tag, String message) {
try {
DocCommentTree tree = env.getDocTrees().getDocCommentTree(el);
TreePath treePath = env.getDocTrees().getPath(el);
DocTreePath path = DocTreePath.getPath(treePath, tree, tag);
if (path != null) {
log.print(Diagnostic.Kind.WARNING, path, message);
}
else {
log.print(Diagnostic.Kind.WARNING, el, message);
}
}
catch (Throwable t) {
t.printStackTrace();
}
}
/**
* Logs an error with the provided message
*
* @param el the current element
* @param tag the current tag
* @param message the message
*/
final void logError(Element el, DocTree tag, String message) {
try {
DocCommentTree tree = env.getDocTrees().getDocCommentTree(el);
TreePath treePath = env.getDocTrees().getPath(el);
DocTreePath path = DocTreePath.getPath(treePath, tree, tag);
if (path != null) {
log.print(Diagnostic.Kind.ERROR, path, message);
}
else {
log.print(Diagnostic.Kind.ERROR, el, message);
}
}
catch (Throwable t) {
t.printStackTrace();
}
}
final DocTrees getDocTrees() {
return env.getDocTrees();
}
final Elements getElementUtils() {
return env.getElementUtils();
}
/**
* Gets a mapping of the provided list of attributes
*
* @param attributes the attributes list
* @return the attributes mapping
*/
Map<String, String> getAttributes(Element el, List<? extends DocTree> attributes) {
return attributes
.stream()
.filter(AttributeTree.class::isInstance)
.map(AttributeTree.class::cast)
.collect(Collectors.toMap(attr -> attr.getName().toString().toLowerCase(),
attr -> attr.getValue() != null ? convertTree(el, attr.getValue()) : ""));
}
/**
* Aligns the lines in the provided text to the same indentation level
*
* @param text the text
* @return the new text all aligned to the same indentation level
*/
static String alignIndent(String text) {
int index = text.indexOf('\n');
if (index == -1) {
return text;
}
StringBuilder builder = new StringBuilder();
return builder.append(text.substring(0, index + 1))
.append(text.substring(index + 1).stripIndent())
.toString();
}
}
@@ -0,0 +1,220 @@
package ghidra.doclets.typestubs;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Set;
import javax.lang.model.element.PackageElement;
import javax.lang.model.element.TypeElement;
import javax.lang.model.element.VariableElement;
import javax.lang.model.util.Elements;
/**
* A builder class for the pseudo ghidra.ghidra_builtins package
*/
class GhidraBuiltinsBuilder {
private static final String INDENT = "";
private final PythonTypeStubDoclet doclet;
private final PythonTypeStubType api;
private final PythonTypeStubType script;
/**
* Creates a new {@link GhidraBuiltinsBuilder}
*
* @param doclet the current doclet
*/
GhidraBuiltinsBuilder(PythonTypeStubDoclet doclet) {
this.doclet = doclet;
this.api = getType(doclet, "ghidra.program.flatapi.FlatProgramAPI");
this.script = getType(doclet, "ghidra.app.script.GhidraScript");
}
/**
* Processes the pseudo ghidra.ghidra_builtins package
*/
void process() {
File root = new File(doclet.getDestDir(), "ghidra-stubs/ghidra_builtins");
root.mkdirs();
File stub = new File(root, "__init__.pyi");
try (PrintWriter printer = new PrintWriter(new FileWriter(stub))) {
process(printer);
}
catch (IOException e) {
e.printStackTrace();
}
}
/**
* Processes the pseudo ghidra.ghidra_builtins package using the provided printer
*
* @param printer the printer
*/
private void process(PrintWriter printer) {
// collect methods and fields early to ensure protected visibility
api.getMethods(true, true);
script.getMethods(true, true);
api.getFields(true);
script.getFields(true);
script.writeJavaDoc(printer, INDENT);
printer.println();
printScriptImports(printer);
printTypeVars(printer);
// we need to keep track of things to export for __all__
Set<String> exports = new LinkedHashSet<>();
printFields(printer, exports);
printer.println();
printer.println();
printMethods(printer, exports);
printer.print("__all__ = [");
printer.print(String.join(", ", exports));
printer.println("]");
}
/**
* Prints all necessary TypeVars
*
* @param printer the printer
*/
private void printTypeVars(PrintWriter printer) {
for (String typevar : getScriptTypeVars()) {
printer.print(typevar);
printer.print(" = typing.TypeVar(\"");
printer.print(typevar);
printer.println("\")");
}
printer.println();
printer.println();
}
/**
* Prints all the script fields
*
* @param printer the printer
* @param exports the set of fields to export
*/
private void printFields(PrintWriter printer, Set<String> exports) {
// always use false for static so typing.ClassVar is not emitted
for (VariableElement field : api.getFields(true)) {
api.printField(field, printer, INDENT, false);
exports.add('"' + field.getSimpleName().toString() + '"');
}
for (VariableElement field : script.getFields(true)) {
script.printField(field, printer, INDENT, false);
exports.add('"' + field.getSimpleName().toString() + '"');
}
}
/**
* Prints all the script methods
*
* @param printer the printer
* @param exports the set of methods to export
*/
private void printMethods(PrintWriter printer, Set<String> exports) {
// methods must be sorted by name for typing.overload
List<PythonTypeStubMethod> apiMethods = api.getMethods(true, true);
List<PythonTypeStubMethod> scriptMethods = script.getMethods(true, true);
int length = apiMethods.size() + scriptMethods.size();
List<PythonTypeStubMethod> methods = new ArrayList<>(length);
methods.addAll(apiMethods);
methods.addAll(scriptMethods);
methods.sort(null);
ListIterator<PythonTypeStubMethod> methodIterator = methods.listIterator();
while (methodIterator.hasNext()) {
PythonTypeStubMethod method = methodIterator.next();
boolean overload = PythonTypeStubType.isOverload(methods, methodIterator, method);
method.process(printer, INDENT, overload);
exports.add('"' + method.getName() + '"');
printer.println();
}
}
/**
* Gets a list of all imported packages
*
* @return the list of packages
*/
private List<PackageElement> getScriptPackages() {
Set<PackageElement> packages = new HashSet<>();
for (TypeElement type : api.getImportedTypes()) {
packages.add(PythonTypeStubElement.getPackage(type));
}
for (TypeElement type : script.getImportedTypes()) {
packages.add(PythonTypeStubElement.getPackage(type));
}
List<PackageElement> res = new ArrayList<>(packages);
res.sort(PythonTypeStubElement::compareQualifiedNameable);
return res;
}
/**
* Prints the imports needed by this package
*
* @param printer the printer
*/
private void printScriptImports(PrintWriter printer) {
printer.println("import collections.abc");
printer.println("import typing");
printer.println("from warnings import deprecated # type: ignore");
printer.println();
printer.println("import jpype # type: ignore");
printer.println("import jpype.protocol # type: ignore");
printer.println();
doclet.printImports(printer, getScriptPackages());
printer.println();
printer.println();
printer.println("from ghidra.app.script import *");
printer.println();
printer.println();
}
/**
* Gets a list of TypeVars needed by this package
*
* @return the list of TypeVars
*/
private List<String> getScriptTypeVars() {
// all this for only two typing.TypeVar
// at least this is future proof
Set<String> vars = new HashSet<>(api.getTypeVars());
vars.addAll(script.getTypeVars());
List<String> res = new ArrayList<>(vars);
res.sort(null);
return res;
}
/**
* Gets the PythonTypeStubType for the provided type name
*
* @param doclet the current doclet
* @param name the type name
* @return the requested type
*/
private static PythonTypeStubType getType(PythonTypeStubDoclet doclet, String name) {
Elements elements = doclet.getElementUtils();
TypeElement type = elements.getTypeElement(name);
PackageElement pkg = (PackageElement) type.getEnclosingElement();
return new PythonTypeStubType(new PythonTypeStubPackage(doclet, pkg), type);
}
}
@@ -0,0 +1,494 @@
package ghidra.doclets.typestubs;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import javax.lang.model.element.Element;
import com.sun.source.doctree.DocTree;
import com.sun.source.doctree.EndElementTree;
import com.sun.source.doctree.LinkTree;
import com.sun.source.doctree.StartElementTree;
import com.sun.source.doctree.TextTree;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
/**
* Helper class for converting HTML to reStructuredText
*/
public final class HtmlConverter extends DocConverter {
private final JavadocConverter docConverter;
/**
* Creates a new {@link HtmlConverter}
*
* @param env the doclet environment
* @param log the log
*/
public HtmlConverter(DocletEnvironment env, Reporter log, JavadocConverter docConverter) {
super(env, log);
this.docConverter = docConverter;
}
@Override
String convertTag(Element el, DocTree tag, ListIterator<? extends DocTree> it) {
return docConverter.convertTag(el, tag, it);
}
/**
* Gets a map of the attributes in the html element
*
* @param start the start element
* @return the attributes map
*/
public Map<String, String> getAttributes(Element el, StartElementTree start) {
return getAttributes(el, start.getAttributes());
}
/**
* Logs a warning about an unterminated html tag
*
* @param el the current element
* @param tag the current tag
*/
public void logUnterminatedHtml(Element el, StartElementTree tag) {
try {
logWarning(el, tag, "unterminated html tag");
}
catch (Throwable t) {
t.printStackTrace();
}
}
/**
* Converts the provided HTML to reStructuredText where possible
*
* @param tag the html
* @param el the element containing the html
* @param it the Javadoc tree iterator
* @return the converted string
*/
String convertHtml(HtmlDocTree tag, Element el, ListIterator<? extends DocTree> it) {
StartElementTree start = tag.getStartTag();
return switch (tag.getHtmlKind()) {
case A -> convertAnchor(tag, el);
case B -> "**" + convertTree(el, tag.getBody()) + "**";
case BIG -> ""; // not in rst
case BLOCKQUOTE -> convertBlockQuote(tag, el);
case BR -> "\n";
case CAPTION -> {
logError(el, start, "<caption> outside of table");
yield start.toString();
}
case CITE -> "*" + convertTree(el, tag.getBody()) + "*";
case CODE -> "``" + convertTree(el, tag.getBody()) + "``";
case DD -> {
logError(el, start, "<dd> outside of list");
yield start.toString();
}
case DEL -> "~~" + convertTree(el, tag.getBody()) + "~~";
// rarely used, not bothering with id attribute
case DFN -> "*" + convertTree(el, tag.getBody()) + "*";
case DIV -> convertTree(el, tag.getBody()); // do nothing
case DL -> convertDescriptionList(tag, el);
case DT -> {
logError(el, start, "<dt> outside of list");
yield start.toString();
}
case EM -> "*" + convertTree(el, tag.getBody()) + "*";
case H1 -> convertHeader(tag, el, '#');
case H2 -> convertHeader(tag, el, '*');
case H3 -> convertHeader(tag, el, '=');
case H4 -> convertHeader(tag, el, '-');
case H5 -> convertHeader(tag, el, '^');
case H6 -> convertHeader(tag, el, '\'');
case HR -> "---\n";
case I -> "*" + convertTree(el, tag.getBody()) + "*";
case IMG -> ""; // not supported because the images wouldn't be available
case INS -> convertTree(el, tag.getBody()); // no underline in rst
case LI -> {
logError(el, start, "<li> outside of list");
yield start.toString();
}
case OL -> convertOrderedList(tag, el);
case P -> "\n";
case PRE -> convertTree(el, tag.getBody()); // do nothing
case SMALL -> ""; // not in rst
case SPAN -> convertTree(el, tag.getBody()); // no colored text in rst
case STRONG -> "**" + convertTree(el, tag.getBody()) + "**";
case SUB -> ""; // no subscript in rst
case SUP -> ""; // no superscript in rst
case TABLE -> convertTable(tag, el);
case TBODY -> {
logError(el, start, "<tbody> outside of table");
yield start.toString();
}
case TD -> {
logError(el, start, "<td> outside of table");
yield start.toString();
}
case TFOOT -> {
logError(el, start, "<tfoot> outside of table");
yield start.toString();
}
case TH -> {
logError(el, start, "<th> outside of table");
yield start.toString();
}
case THEAD -> {
logError(el, start, "<thead> outside of table");
yield start.toString();
}
case TR -> {
logError(el, start, "<tr> outside of table");
yield start.toString();
}
case TT -> "``" + convertTree(el, tag.getBody()) + "``";
case U -> convertTree(el, tag.getBody()); // no underline in rst
case UL -> convertUnorderedList(tag, el);
case UNSUPPORTED -> {
logWarning(el, start, "unsupported html tag");
yield start.toString();
}
case VAR -> "*" + convertTree(el, tag.getBody()) + "*";
};
}
String convertHtml(StartElementTree start, Element el, ListIterator<? extends DocTree> it) {
HtmlDocTree tag = HtmlDocTree.getTree(this, start, el, it);
return convertHtml(tag, el, it);
}
/**
* Converts a {@literal <blockquote>} tag
*
* @param html the html
* @param el the element
* @return the converted blockquote
*/
private String convertBlockQuote(HtmlDocTree html, Element el) {
String body = convertTree(el, html.getBody());
return body.indent(INDENT_WIDTH);
}
/**
* Converts the {@literal <H1>} ... {@literal <H6>} tags
*
* @param html the html
* @param el the element
* @param header the header character
* @return the converted header
*/
private String convertHeader(HtmlDocTree html, Element el, char header) {
String body = convertTree(el, html.getBody());
int length = body.length();
StringBuilder builder = new StringBuilder();
return builder.append('\n')
.repeat(header, length)
.append('\n')
.append(body)
.append('\n')
.repeat(header, length)
.append('\n')
.toString();
}
/**
* Converts a {@literal <li>} tag
*
* @param tree the html
* @param el the element
* @return the converted list entry
*/
private String convertListEntry(HtmlDocTree tree, Element el) {
StringBuilder builder = new StringBuilder();
for (DocTree tag : tree.getBody()) {
if (tag instanceof HtmlDocTree html) {
switch (html.getHtmlKind()) {
case OL: {
String list = convertOrderedList(html, el);
builder.append(list.indent(INDENT_WIDTH));
break;
}
case UL: {
String list = convertUnorderedList(html, el);
builder.append(list.indent(INDENT_WIDTH));
break;
}
default: {
builder.append(convertTree(el, html.getBody()));
break;
}
}
}
else {
String entry = docConverter.convertTag(el, tag, null);
builder.append(alignIndent(entry));
}
}
return builder.toString();
}
/**
* Converts a description list {@literal <dl>}
*
* @param tree the html
* @param el the element
* @return the converted list
*/
private String convertDescriptionList(HtmlDocTree tree, Element el) {
StringBuilder builder = new StringBuilder();
builder.append('\n');
for (DocTree tag : tree.getBody()) {
if (tag instanceof HtmlDocTree html) {
if (html.getHtmlKind() == HtmlTagKind.DT) {
builder.append(convertTree(el, html.getBody()));
}
else if (html.getHtmlKind() == HtmlTagKind.DD) {
String body = convertTree(el, html.getBody());
builder.append(body.indent(INDENT_WIDTH))
.append('\n');
}
else {
builder.append(convertTree(el, html.getBody()));
}
}
else {
builder.append(docConverter.convertTag(el, tag, null));
}
}
return builder.toString();
}
/**
* Converts an ordered list {@literal <ol>}
*
* @param tree the html
* @param el the element
* @return the converted list
*/
private String convertOrderedList(HtmlDocTree tree, Element el) {
StringBuilder builder = new StringBuilder();
int num = 1; // because #. doesn't always work like it should
builder.append('\n');
for (DocTree tag : tree.getBody()) {
if (tag instanceof HtmlDocTree html) {
if (html.getHtmlKind() == HtmlTagKind.LI) {
builder.append(num++)
.append(". ")
.append(convertListEntry(html, el))
.append('\n');
}
else {
builder.append(convertTree(el, html.getBody()));
}
}
else {
builder.append(docConverter.convertTag(el, tag, null));
}
}
return builder.toString();
}
/**
* Converts an unordered list {@literal <ul>}
*
* @param tree the html
* @param el the element
* @return the converted list
*/
private String convertUnorderedList(HtmlDocTree tree, Element el) {
StringBuilder builder = new StringBuilder();
builder.append('\n');
for (DocTree tag : tree.getBody()) {
if (tag instanceof HtmlDocTree html) {
if (html.getHtmlKind() == HtmlTagKind.LI) {
builder.append("* ")
.append(convertListEntry(html, el))
.append('\n');
}
else {
builder.append(convertTree(el, html.getBody()));
}
}
else {
builder.append(docConverter.convertTag(el, tag, null));
}
}
return builder.toString();
}
/**
* Converts an anchor {@literal <a id="#example">link text</a>}
*
* @param html the html
* @param el the element
* @return the converted html
*/
private String convertAnchor(HtmlDocTree html, Element el) {
String label = convertTree(el, html.getBody()).stripLeading();
Map<String, String> attrs = getAttributes(el, html.getStartTag());
String id = attrs.get("id");
if (id == null) {
id = attrs.get("name");
}
if (id != null) {
return "\n.. _" + id + ":\n\n" + label;
}
String href = attrs.get("href");
if (href == null) {
logWarning(el, html.getStartTag(), "skipping anchor without an id or href");
return "";
}
if (href.startsWith("#")) {
// internal
if (label.isBlank()) {
return href.substring(1) + '_';
}
return '`' + label + " <" + href.substring(1) + "_>`_";
}
// external
if (label.isBlank()) {
return '<' + href.substring(0) + '>';
}
return '`' + label + " <" + href + ">`_";
}
/**
* Converts the provided tree to a raw html string
*
* @param el the element
* @param tree the tree
* @return the html string
*/
private String getRawHtml(Element el, List<? extends DocTree> tree) {
StringBuilder builder = new StringBuilder();
for (DocTree tag : tree) {
switch (tag.getKind()) {
case START_ELEMENT:
case END_ELEMENT:
builder.append(tag.toString());
break;
case OTHER:
if (!(tag instanceof HtmlDocTree)) {
logError(el, tag, "Unexpected OTHER tag kind");
return "";
}
HtmlDocTree html = (HtmlDocTree) tag;
builder.append(html.getStartTag().toString())
.append(getRawHtml(el, html.getBody()));
EndElementTree end = html.getEndTag();
if (end != null) {
builder.append(end.toString());
}
break;
case LINK:
case LINK_PLAIN:
builder.append(getRawHtml(el, ((LinkTree) tag).getLabel()));
break;
default:
builder.append(docConverter.convertTag(el, tag, null));
break;
}
}
return builder.toString();
}
/**
* Converts the html tree to a raw html string
*
* @param html the html tree
* @param el the element
* @return the html
*/
private String getRawHtml(HtmlDocTree html, Element el) {
StringBuilder builder = new StringBuilder();
builder.append(html.getStartTag().toString())
.append(getRawHtml(el, html.getBody()));
EndElementTree end = html.getEndTag();
if (end != null) {
builder.append(end.toString());
}
return builder.toString();
}
/**
* Converts a table {@literal <table>} to reStructuredText if possible
*
* @param tree the html
* @param el the element
* @return the converted table or original html if not convertible
*/
private String convertTable(HtmlDocTree tree, Element el) {
try {
return tryConvertTable(tree, el);
}
catch (UnsupportedOperationException e) {
// use raw html directive
// this may not be supported by all IDEs but it is better then nothing
// if your IDE doesn't support it, try tilting your head and squinting
StringBuilder builder = new StringBuilder();
return builder.append("\n\n.. raw:: html\n\n")
.append(getRawHtml(tree, el).indent(INDENT_WIDTH))
.append('\n')
.toString();
}
}
/**
* Converts a table {@literal <table>}
*
* @param tree the html
* @param el the element
* @return the converted table
* @throws UnsupportedOperationException if the table contains nested rows
*/
private String tryConvertTable(HtmlDocTree tree, Element el) {
RstTableBuilder tbl = new RstTableBuilder(this, el);
ListIterator<? extends DocTree> it = tree.getBody().listIterator();
while (it.hasNext()) {
DocTree tag = it.next();
switch (tag.getKind()) {
case OTHER:
if (!(tag instanceof HtmlDocTree)) {
logError(el, tag, "Unexpected OTHER tag kind");
return "";
}
HtmlDocTree html = (HtmlDocTree) tag;
switch (html.getHtmlKind()) {
case TBODY:
case TFOOT:
case THEAD:
tbl.addRowGroup(html);
break;
case TR:
tbl.addRow(html);
break;
case CAPTION:
tbl.addCaption(convertTree(el, html.getBody()));
break;
default:
logError(el, tag,
"unexpected html tag encountered while parsing table");
break;
}
break;
case TEXT:
String body = ((TextTree) tag).getBody();
if (!body.isBlank()) {
logWarning(el, tag, "skipping unexpected text in table");
}
break;
default:
logError(el, tag, "unexpected tag encountered while parsing table");
return "";
}
}
return tbl.build();
}
}
@@ -0,0 +1,150 @@
package ghidra.doclets.typestubs;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
import javax.lang.model.element.Element;
import com.sun.source.doctree.DocTree;
import com.sun.source.doctree.DocTreeVisitor;
import com.sun.source.doctree.EndElementTree;
import com.sun.source.doctree.StartElementTree;
import com.sun.source.doctree.TextTree;
/**
* A {@link DocTree} for handling HTML<p/>
*
* This class allows for converting the HTML tags recursively in the same fashion
* as the Javadoc tags.
*/
public final class HtmlDocTree implements DocTree {
private final HtmlTagKind kind;
private final StartElementTree start;
private final EndElementTree end;
private final List<? extends DocTree> body;
/**
* Gets an {@link HtmlDocTree} for the provided {@link StartElementTree}
*
* @param converter the html converter
* @param start the html start
* @param el the element containing the documentation being processed
* @param it the iterator over the remaining tags
* @return the created {@link HtmlDocTree}
*/
public static HtmlDocTree getTree(HtmlConverter converter, StartElementTree start, Element el,
ListIterator<? extends DocTree> it) {
HtmlTagKind kind = HtmlTagKind.getKind(start);
List<DocTree> body = new ArrayList<>();
if (start.isSelfClosing() || HtmlTagKind.isVoidTag(kind)) {
return new HtmlDocTree(kind, start, null, body);
}
while (it.hasNext()) {
DocTree tag = it.next();
switch (tag.getKind()) {
case START_ELEMENT:
if (kind.isTerminateBy((StartElementTree) tag)) {
// hack for unclosed elements
it.previous();
converter.logUnterminatedHtml(el, start);
return new HtmlDocTree(kind, start, null, body);
}
body.add(HtmlDocTree.getTree(converter, (StartElementTree) tag, el, it));
break;
case END_ELEMENT:
if (kind.isTerminateBy((EndElementTree) tag)) {
// hack for unclosed elements
it.previous();
converter.logUnterminatedHtml(el, start);
return new HtmlDocTree(kind, start, null, body);
}
if (kind == HtmlTagKind.getKind((EndElementTree) tag)) {
return new HtmlDocTree(kind, start, (EndElementTree) tag, body);
}
body.add(tag);
break;
case TEXT:
String text = ((TextTree) tag).getBody();
if (kind != HtmlTagKind.PRE && text.isBlank()) {
continue;
}
body.add(tag);
break;
default:
body.add(tag);
break;
}
}
converter.logUnterminatedHtml(el, start);
return new HtmlDocTree(kind, start, null, body);
}
/**
* Creates a new {@link HtmlDocTree} with the provided fields
*
* @param kind the html tag kind
* @param start the start element
* @param end the optional end element
* @param body the html body
*/
private HtmlDocTree(HtmlTagKind kind, StartElementTree start, EndElementTree end,
List<DocTree> body) {
this.kind = kind;
this.start = start;
this.end = end;
this.body = Collections.unmodifiableList(body);
}
@Override
public Kind getKind() {
// OTHER is implementation reserved
// Since this is implementation specific, lets use it
return Kind.OTHER;
}
@Override
public <R, D> R accept(DocTreeVisitor<R, D> visitor, D data) {
throw new UnsupportedOperationException();
}
/**
* Gets the html body
*
* @return the html body
*/
public List<? extends DocTree> getBody() {
return body;
}
/**
* Gets the html tag kind
*
* @return the html tag kind
*/
public HtmlTagKind getHtmlKind() {
return kind;
}
/**
* Gets the html start element tree
*
* @return the html start element
*/
public StartElementTree getStartTag() {
return start;
}
/**
* Gets the html end element tree<p/>
*
* This may be null if the html tag is a "void" tag or if the html is malformed
*
* @return the html end element or null
*/
public EndElementTree getEndTag() {
return end;
}
}
@@ -0,0 +1,351 @@
package ghidra.doclets.typestubs;
import java.util.HashMap;
import java.util.Map;
import com.sun.source.doctree.EndElementTree;
import com.sun.source.doctree.StartElementTree;
public enum HtmlTagKind {
// This would be much simpler if we didn't have to handle malformed html
// HTML container tags REQUIRE a closing tag
// Unfortunately they are often ommitted, even in the JDK API, which makes
// this much more complicated then it needs to be.
// Best we can do it try not to consume elements that can't possibly be ours,
// log it when encountered and then hope the result isn't ruined.
A,
B,
BIG,
BLOCKQUOTE {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return kind == this;
}
},
BR,
CAPTION,
CITE,
CODE,
DD {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case DD, DT, DL -> true;
default -> false;
};
}
},
DEL,
DFN,
DIV,
DL {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case DL -> true;
default -> false;
};
}
},
DT {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case DD, DT, DL -> true;
default -> false;
};
}
},
EM,
H1 {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
if (isInline(kind)) {
return false;
}
return switch (kind) {
case A -> false;
default -> true;
};
}
},
H2 {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
if (isInline(kind)) {
return false;
}
return switch (kind) {
case A -> false;
default -> true;
};
}
},
H3 {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
if (isInline(kind)) {
return false;
}
return switch (kind) {
case A -> false;
default -> true;
};
}
},
H4 {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
if (isInline(kind)) {
return false;
}
return switch (kind) {
case A -> false;
default -> true;
};
}
},
H5 {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
if (isInline(kind)) {
return false;
}
return switch (kind) {
case A -> false;
default -> true;
};
}
},
H6 {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
if (isInline(kind)) {
return false;
}
return switch (kind) {
case A -> false;
default -> true;
};
}
},
HR,
I,
IMG,
INS,
LI {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case LI -> true;
default -> false;
};
}
@Override
public boolean isTerminateBy(EndElementTree end) {
return switch (getKind(end)) {
case OL, UL -> true;
default -> false;
};
}
},
OL {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return false;
}
},
P,
PRE {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return false;
}
},
SMALL,
SPAN,
STRONG,
SUB,
SUP,
TABLE {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
// no nested tables
return kind == this;
}
},
TBODY {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case THEAD, TFOOT, TBODY, TABLE -> true;
default -> false;
};
}
},
TD {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case TD, TH, TR, THEAD, TFOOT, TBODY, TABLE -> true;
default -> false;
};
}
},
TFOOT {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case THEAD, TFOOT, TBODY, TABLE -> true;
default -> false;
};
}
},
TH {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case TD, TH, TR, THEAD, TFOOT, TBODY, TABLE -> true;
default -> false;
};
}
},
THEAD {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case THEAD, TFOOT, TBODY, TABLE -> true;
default -> false;
};
}
},
TR {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return switch (kind) {
case TR, TABLE, THEAD, TFOOT, TBODY -> true;
default -> false;
};
}
},
TT,
U,
UL {
@Override
boolean isTerminateBy(HtmlTagKind kind) {
return false;
}
},
VAR,
UNSUPPORTED;
private static final Map<String, HtmlTagKind> LOOKUP;
static {
HtmlTagKind[] values = values();
LOOKUP = new HashMap<>(values.length);
for (HtmlTagKind value : values) {
LOOKUP.put(value.name(), value);
}
}
/**
* Gets the HtmlTagKind with the provided name
*
* @param name the name
* @return the HtmlTagKind with the same name or UNSUPPORTED
*/
static HtmlTagKind getKind(String name) {
return LOOKUP.getOrDefault(name, UNSUPPORTED);
}
/**
* Gets the HtmlTagKind for the provided element
*
* @param tag the tag
* @return the HtmlTagKind for the provided tag or UNSUPPORTED
*/
static HtmlTagKind getKind(StartElementTree tag) {
return getKind(tag.getName().toString().toUpperCase());
}
/**
* Gets the HtmlTagKind for the provided element
*
* @param tag the tag
* @return the HtmlTagKind for the provided tag or UNSUPPORTED
*/
static HtmlTagKind getKind(EndElementTree tag) {
return getKind(tag.getName().toString().toUpperCase());
}
/**
* Checks if this tag is terminated by another tag because it can't possibly contain it
*
* @param kind the other HtmlTagKind
* @return true if this tag canot possibly contain the other kind
*/
boolean isTerminateBy(HtmlTagKind kind) {
return !isInline(kind);
}
/**
* Checks if this tag is terminated by another element because it can't possibly contain it
*
* @param kind the other HtmlTagKind
* @return true if this tag canot possibly contain the other element
*/
public final boolean isTerminateBy(StartElementTree start) {
HtmlTagKind kind = getKind(start);
return isTerminateBy(kind);
}
/**
* Checks if this tag is terminated by the closing another element.<p/>
*
* This is usually because the other element would contain it.
*
* @param kind the other HtmlTagKind
* @return true if this tag canot possibly contain the other kind
*/
public boolean isTerminateBy(EndElementTree end) {
HtmlTagKind kind = getKind(end);
if (kind == this) {
// this tag may not be for the current node so we return false here
return false;
}
return isTerminateBy(kind);
}
/**
* Checks if the provided tag is a void or empty tag
*
* @param kind the tag kind
* @return true if this is a void or empty tag
*/
public static boolean isVoidTag(HtmlTagKind kind) {
// technically <p> is NOT a void tag
// unfortunately it is misused so often that the errors/warnings
// would become junk because the <p> tags would have consumed too much
return switch (kind) {
case BR, HR, P -> true;
default -> false;
};
}
/**
* Checks if the provided tag is for inline markup
*
* @param kind the tag kind
* @return true if this kind is for inline markup
*/
public static boolean isInline(HtmlTagKind kind) {
return switch (kind) {
case B, BIG, CITE, DFN, CODE, DEL, EM, I, INS -> true;
case SMALL, STRONG, SUB, SUP, TT, U, VAR -> true;
default -> false;
};
}
}
@@ -0,0 +1,428 @@
package ghidra.doclets.typestubs;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import javax.lang.model.element.Element;
import javax.lang.model.element.ElementKind;
import javax.lang.model.element.Modifier;
import javax.lang.model.element.Name;
import javax.lang.model.element.PackageElement;
import javax.lang.model.element.QualifiedNameable;
import javax.lang.model.element.TypeElement;
import javax.lang.model.type.ArrayType;
import javax.lang.model.type.DeclaredType;
import javax.lang.model.type.TypeMirror;
import javax.lang.model.type.WildcardType;
/**
* Base class providing access to sanitized names (Python safe).
*/
abstract class PythonTypeStubElement<T extends Element> {
private static final Set<String> PY_KEYWORDS = new HashSet<>(
Set.of("False", "None", "True", "and", "as", "assert", "async", "await", "break",
"class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for",
"from", "global", "if", "import", "in", "is", "lambda",
"nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with",
"yield"));
static final String DOC_QUOTES = "\"\"\"";
static final String ALT_DOC_QUOTES = "'''";
static final String PY_INDENT = " ";
final PythonTypeStubDoclet doclet;
final T el;
private final PackageElement pkg;
private String name;
PythonTypeStubElement(PythonTypeStubDoclet doclet, T el) {
this(doclet, getPackage(el), el);
}
PythonTypeStubElement(PythonTypeStubDoclet doclet, PackageElement pkg, T el) {
this.doclet = doclet;
this.pkg = pkg;
this.el = el;
}
/**
* Gets the package for the provided element
*
* @param el the element
* @return the package
*/
static PackageElement getPackage(Element el) {
while (!(el instanceof PackageElement)) {
el = el.getEnclosingElement();
}
return (PackageElement) el;
}
static int compareQualifiedNameable(QualifiedNameable a, QualifiedNameable b) {
return a.getQualifiedName().toString().compareTo(b.getQualifiedName().toString());
}
/**
* Checks if the provided element is in the same package as this element
*
* @param el the other element
* @return true if the other element is declared in the same package
*/
boolean isSamePackage(Element el) {
return pkg.equals(getPackage(el));
}
/**
* Checks if the provided type is in the same package as this element
*
* @param type the type
* @return true if the type is declared in the same package
*/
boolean isSamePackage(TypeMirror type) {
if (type instanceof DeclaredType dt) {
return pkg.equals(getPackage(dt.asElement()));
}
return false;
}
/**
* Gets the type string for the provided type and quotes if necessary<p/>
*
* This string value is safe to be used as a parameter or return type
* as well as for use in a generic type.
*
* @param self the type to become typing.Self if encountered
* @param type the type to get the string for
* @return the type string
*/
String getTypeString(Element self, TypeMirror type) {
String typeName = sanitizeQualifiedName(self, type);
if (isSamePackage(type) && !typeName.equals("typing.Self")) {
typeName = '"' + typeName + '"';
}
return typeName;
}
/**
* Gets the Python safe name for this element
*
* @return the python safe name
*/
final String getName() {
if (name == null) {
name = sanitize(el.getSimpleName());
}
return name;
}
/**
* Writes the Javadoc for the provided element to the provided printer
*
* @param element the element to write the javadoc for
* @param printer the printer to write to
* @param indent the indentation
* @param emptyValue the value to use when there is no documentation
* @return true if a Javadoc was written else false
*/
final boolean writeJavaDoc(Element element, PrintWriter printer, String indent,
String emptyValue) {
String doc = doclet.getJavadoc(element);
if (doc.isBlank()) {
if (!emptyValue.isBlank()) {
printer.print(indent);
printer.print(emptyValue);
}
return false;
}
String quotes = doc.contains(DOC_QUOTES) ? ALT_DOC_QUOTES : DOC_QUOTES;
if (quotes == ALT_DOC_QUOTES) {
// ensure there are no problems
doc = doc.replaceAll(ALT_DOC_QUOTES, '\\' + ALT_DOC_QUOTES);
}
printer.print(indent);
printer.println(quotes);
writeLines(printer, doc.stripTrailing(), indent);
printer.print(indent);
printer.println(quotes);
return true;
}
/**
* Writes the Javadoc for this element to the provided printer
*
* @param printer the printer to write to
* @param indent the indentation
* @param emptyValue the value to use when there is no documentation
*/
final void writeJavaDoc(PrintWriter printer, String indent, String emptyValue) {
writeJavaDoc(el, printer, indent, emptyValue);
}
/**
* Writes the Javadoc for this element to the provided printer
*
* @param printer the printer to write to
* @param indent the indentation
*/
final void writeJavaDoc(PrintWriter printer, String indent) {
writeJavaDoc(el, printer, indent, "");
}
/**
* Makes the provided String Python safe if necessary
*
* @param value the value to make Python safe
* @return the Python safe value
*/
static String sanitize(String value) {
if (PY_KEYWORDS.contains(value)) {
return value + "_";
}
return value;
}
/**
* Makes the provided element name Python safe if necessary
*
* @param name the name to make Python safe
* @return the Python safe name
*/
static String sanitize(Name name) {
return sanitize(name.toString());
}
/**
* Makes the provided qualified name Python safe if necessary
*
* @param name the qualified name to make Python safe
* @return the Python safe qualified name
*/
static String sanitizeQualifiedName(String name) {
Iterator<String> it = Arrays.stream(name.split("\\."))
.map(PythonTypeStubElement::sanitize)
.iterator();
return String.join(".", (Iterable<String>) () -> it);
}
/**
* Makes the provided qualified name Python safe if necessary
*
* @param name the qualified name to make Python safe
* @return the Python safe qualified name
*/
static String sanitizeQualifiedName(QualifiedNameable name) {
return sanitizeQualifiedName(name.getQualifiedName().toString());
}
/**
* Makes the provided package name Python safe if necessary
*
* @param pkg the package to make Python safe
* @return the Python safe package name
*/
static String sanitizeQualifiedName(PackageElement pkg) {
return sanitizeQualifiedName(pkg.getQualifiedName().toString());
}
/**
* Makes the provided type Python safe if necessary
*
* @param self the type to become typing.Self if encountered
* @param type the type to make Python safe
* @param pkg the current package
* @return the Python safe type name
*/
static String sanitize(Element self, TypeMirror type, PackageElement pkg) {
return switch (type.getKind()) {
case DECLARED -> throw new RuntimeException(
"declared types should use the qualified name");
case ARRAY -> {
TypeMirror component = ((ArrayType) type).getComponentType();
yield "jpype.JArray[" + sanitizeQualifiedName(self, component, pkg) + "]";
}
case BOOLEAN -> "jpype.JBoolean";
case BYTE -> "jpype.JByte";
case CHAR -> "jpype.JChar";
case DOUBLE -> "jpype.JDouble";
case FLOAT -> "jpype.JFloat";
case INT -> "jpype.JInt";
case LONG -> "jpype.JLong";
case SHORT -> "jpype.JShort";
case TYPEVAR -> type.toString();
case WILDCARD -> getWildcardVarName(self, (WildcardType) type, pkg);
default -> throw new RuntimeException("unexpected TypeKind " + type.getKind());
};
}
/**
* Checks if the provided type is the same as the provided element
*
* @param self the element of the type to become typing.Self
* @param type the type to check
* @return true if the inputs represent the same type
*/
static final boolean isSelfType(Element self, TypeMirror type) {
if (self.getKind() == ElementKind.ENUM) {
// typing.Self is usually invalid here
return false;
}
if (type instanceof DeclaredType dt) {
return self.equals(dt.asElement());
}
return false;
}
/**
* Makes the qualified name for the provided type Python safe if necessary
*
* @param self the type to become typing.Self if encountered
* @param type the type to make Python safe
* @return the Python safe qualified type name
*/
final String sanitizeQualifiedName(Element self, TypeMirror type) {
return sanitizeQualifiedName(self, type, pkg);
}
/**
* Makes the qualified name for the provided type Python safe if necessary<p/>
*
* The provided package is used to check each type and generic components.
* If they require a "forward declaration", it is handled accordingly.
*
* @param self the type to become typing.Self if encountered
* @param type the type to make Python safe
* @param pkg the current package
* @return the Python safe qualified type name
*/
static final String sanitizeQualifiedName(Element self, TypeMirror type, PackageElement pkg) {
if (isSelfType(self, type)) {
return "typing.Self";
}
if (type instanceof DeclaredType dt) {
TypeElement el = (TypeElement) dt.asElement();
PackageElement typePkg = getPackage(el);
String name;
if (pkg.equals(typePkg)) {
name = sanitize(el.getSimpleName());
Element parent = el.getEnclosingElement();
while (parent instanceof TypeElement parentType) {
parent = parent.getEnclosingElement();
name = sanitize(parentType.getSimpleName()) + "." + name;
}
}
else {
name = sanitizeQualifiedName(el);
}
List<? extends TypeMirror> args = dt.getTypeArguments();
if (args.isEmpty()) {
return name;
}
Iterable<String> it = () -> args.stream()
.map(paramType -> sanitizeQualifiedName(self, paramType, pkg))
.iterator();
return name + "[" + String.join(", ", it) + "]";
}
return sanitize(self, type, pkg);
}
/**
* Checks if the provided element is static
*
* @param el the element to check
* @return true if the element is static
*/
static boolean isStatic(Element el) {
return el.getModifiers().contains(Modifier.STATIC);
}
/**
* Checks if the provided element is final
*
* @param el the element to check
* @return true if the element is final
*/
static boolean isFinal(Element el) {
return el.getModifiers().contains(Modifier.FINAL);
}
/**
* Checks if the provided element is public
*
* @param el the element to check
* @return true if the element is public
*/
static boolean isPublic(Element el) {
return el.getModifiers().contains(Modifier.PUBLIC);
}
/**
* Checks if the provided element is protected
*
* @param el the element to check
* @return true if the element is protected
*/
static boolean isProtected(Element el) {
return el.getModifiers().contains(Modifier.PROTECTED);
}
/**
* Increases the provided indentation by one level
*
* @param indent the indentation
* @return the new indentation
*/
static String indent(String indent) {
return indent + PY_INDENT;
}
/**
* Decreases the provided indentation by one level
*
* @param indent the indentation
* @return the new indentation
*/
static String deindent(String indent) {
return indent.substring(0, indent.length() - PY_INDENT.length());
}
/**
* Gets the name for a wildcard type if possible
*
* @param self the type to become typing.Self if encountered
* @param type the wildcard type
* @param pkg the current package
* @return the determined type name if possible otherwise typing.Any
*/
private static String getWildcardVarName(Element self, WildcardType type, PackageElement pkg) {
TypeMirror base = type.getExtendsBound();
if (base == null) {
base = type.getSuperBound();
}
if (base != null) {
return sanitizeQualifiedName(self, base, pkg);
}
return "typing.Any";
}
/**
* Writes the lines to the printer with the provided intentation
*
* @param printer the printer
* @param lines the lines to write
* @param indent the indentation to use
*/
private static void writeLines(PrintWriter printer, String lines, String indent) {
lines.lines().forEach((line) -> {
printer.print(indent);
printer.println(line);
});
}
}
@@ -0,0 +1,499 @@
package ghidra.doclets.typestubs;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.lang.model.element.Element;
import javax.lang.model.element.ExecutableElement;
import javax.lang.model.element.Modifier;
import javax.lang.model.element.QualifiedNameable;
import javax.lang.model.element.TypeElement;
import javax.lang.model.element.TypeParameterElement;
import javax.lang.model.element.VariableElement;
import javax.lang.model.type.ArrayType;
import javax.lang.model.type.DeclaredType;
import javax.lang.model.type.ExecutableType;
import javax.lang.model.type.TypeKind;
import javax.lang.model.type.TypeMirror;
/**
* {@link PythonTypeStubElement} for a method
*/
final class PythonTypeStubMethod extends PythonTypeStubElement<ExecutableElement>
implements Comparable<PythonTypeStubMethod> {
private static final String EMPTY_DOCS = "..." + System.lineSeparator();
private static final Map<String, String> AUTO_CONVERSIONS = new HashMap<>(
Map.ofEntries(
Map.entry("java.lang.Boolean", "typing.Union[java.lang.Boolean, bool]"),
Map.entry("java.lang.Byte", "typing.Union[java.lang.Byte, int]"),
Map.entry("java.lang.Character", "typing.Union[java.lang.Character, int, str]"),
Map.entry("java.lang.Double", "typing.Union[java.lang.Double, float]"),
Map.entry("java.lang.Float", "typing.Union[java.lang.Float, float]"),
Map.entry("java.lang.Integer", "typing.Union[java.lang.Integer, int]"),
Map.entry("java.lang.Long", "typing.Union[java.lang.Long, int]"),
Map.entry("java.lang.Short", "typing.Union[java.lang.Short, int]"),
Map.entry("java.lang.String", "typing.Union[java.lang.String, str]"),
Map.entry("java.io.File", "jpype.protocol.SupportsPath"),
Map.entry("java.nio.file.Path", "jpype.protocol.SupportsPath"),
Map.entry("java.lang.Iterable", "collections.abc.Sequence"),
Map.entry("java.util.Collection", "collections.abc.Sequence"),
Map.entry("java.util.Map", "collections.abc.Mapping"),
Map.entry("java.time.Instant", "datetime.datetime"),
Map.entry("java.sql.Time", "datetime.time"),
Map.entry("java.sql.Date", "datetime.date"),
Map.entry("java.sql.Timestamp", "datetime.datetime"),
Map.entry("java.math.BigDecimal", "decimal.Decimal")));
// FIXME: list and set aren't automatically converted to java.util.List and java.util.Set :(
// if wanted they could be setup to be converted automatically by pyhidra
// however, when passed as a parameter and modified, the original underlyng python container
// wouldn't be modified. To make it work as expected, a python implementation for
// java.util.List and java.util.Set would need to be created using jpype.JImplements,
// that would wrap the list/set before passing it to Java instead of copying the contents
// into a Java List/Set.
private static final Map<String, String> RESULT_CONVERSIONS = new HashMap<>(
Map.of(
"java.lang.Boolean", "bool",
"java.lang.Byte", "int",
"java.lang.Character", "str",
"java.lang.Double", "float",
"java.lang.Float", "float",
"java.lang.Integer", "int",
"java.lang.Long", "int",
"java.lang.Short", "int",
"java.lang.String", "str"));
private final PythonTypeStubType parent;
private final boolean filterSelf;
List<String> typevars;
Set<TypeElement> imports;
/**
* Creates a new {@link PythonTypeStubMethod}
*
* @param parent the type containing this method
* @param el the element for this method
*/
PythonTypeStubMethod(PythonTypeStubType parent, ExecutableElement el) {
this(parent, el, false);
}
/**
* Creates a new {@link PythonTypeStubMethod}
*
* @param parent the type containing this method
* @param el the element for this method
* @param filterSelf true if the self parameter should be filtered
*/
PythonTypeStubMethod(PythonTypeStubType parent, ExecutableElement el, boolean filterSelf) {
super(parent.doclet, el);
this.parent = parent;
this.filterSelf = filterSelf;
}
/**
* Processes the method and prints it to the provided printer
*
* @param printer the printer
* @param indent the indentation
* @param overload true if the overload annotation should be applied
*/
void process(PrintWriter printer, String indent, boolean overload) {
String name = sanitize(getName());
Set<Modifier> modifiers = el.getModifiers();
boolean isStatic = modifiers.contains(Modifier.STATIC);
if (name.equals("<init>")) {
name = "__init__";
}
printer.print(indent);
if (isStatic) {
printer.println("@staticmethod");
printer.print(indent);
}
if (overload) {
printer.println("@typing.overload");
printer.print(indent);
}
if (doclet.isDeprecated(el)) {
String msg = doclet.getDeprecatedMessage(el);
if (msg != null) {
// a message is required
// if one is not present, don't apply it
printer.print("@deprecated(");
printer.print(msg);
printer.println(')');
printer.print(indent);
}
}
printer.print("def ");
printer.print(name);
printSignature(printer, filterSelf || isStatic);
printer.println(":");
indent += PY_INDENT;
writeJavaDoc(el, printer, indent, EMPTY_DOCS);
printer.println();
}
/**
* Gets a collection of all TypeVars needed by this method
*
* @return a collection of all needed TypeVars
*/
Collection<String> getTypeVars() {
if (typevars != null) {
return typevars;
}
List<? extends TypeParameterElement> params = el.getTypeParameters();
typevars = new ArrayList<>(params.size());
for (TypeParameterElement param : params) {
typevars.add(param.getSimpleName().toString());
}
return typevars;
}
/**
* Gets a collection of all type that need to be imported for this method
*
* @return a collection of types to import
*/
Collection<TypeElement> getImportedTypes() {
if (imports != null) {
return imports;
}
List<? extends VariableElement> parameters = el.getParameters();
TypeMirror resType = el.getReturnType();
// make the set big enough for all paramters and the return type
imports = new HashSet<>(parameters.size() + 1);
if (resType instanceof DeclaredType dt) {
imports.add((TypeElement) dt.asElement());
}
for (VariableElement param : parameters) {
if (param.asType() instanceof DeclaredType dt) {
imports.add((TypeElement) dt.asElement());
}
}
return imports;
}
/**
* Converts the result type to the Python equivalent type if applicable
*
* @param type the result type
* @return the Python equivalent type or null if there is no equivalent type
*/
static String convertResultType(TypeMirror type) {
if (type.getKind().isPrimitive()) {
return switch (type.getKind()) {
case BOOLEAN -> "bool";
case BYTE -> "int";
case CHAR -> "str";
case DOUBLE -> "float";
case FLOAT -> "float";
case INT -> "int";
case LONG -> "int";
case SHORT -> "int";
default -> throw new RuntimeException("unexpected TypeKind " + type.getKind());
};
}
if (type instanceof DeclaredType dt) {
Element element = dt.asElement();
if (element instanceof QualifiedNameable nameable) {
return RESULT_CONVERSIONS.get(nameable.getQualifiedName().toString());
}
}
return null;
}
/**
* Checks if this method is a candidate for a Python property
*
* @return true if this method may be a Python property
*/
boolean isProperty() {
if (isStatic(el)) {
return false;
}
List<? extends VariableElement> params = el.getParameters();
if (params.size() > 1) {
return false;
}
String name = getName();
TypeKind resultKind = getReturnType().getKind();
try {
if (name.startsWith("get")) {
return Character.isUpperCase(name.charAt(3)) && resultKind != TypeKind.VOID;
}
if (name.startsWith("is")) {
return Character.isUpperCase(name.charAt(2)) && resultKind != TypeKind.VOID;
}
if (name.startsWith("set")) {
if (params.size() != 1) {
return false;
}
return Character.isUpperCase(name.charAt(3)) && resultKind == TypeKind.VOID;
}
}
catch (IndexOutOfBoundsException e) {
// name check failed
}
return false;
}
/**
* Converts this method to its Python property form
*
* @return this method as a Python property
*/
PropertyMethod asProperty() {
return new PropertyMethod();
}
/**
* Prints the Python equivalent method signature to the provided printer
*
* @param printer the printer
* @param isStatic true if this method is a static method
*/
private void printSignature(PrintWriter printer, boolean isStatic) {
List<String> names = getParameterNames();
List<? extends TypeMirror> types = getParameterTypes();
StringBuilder args = new StringBuilder();
if (!isStatic) {
args.append("self");
}
for (int i = 0; i < names.size(); i++) {
if (i != 0 || !isStatic) {
args.append(", ");
}
if (el.isVarArgs() && i == names.size() - 1) {
ArrayType type = (ArrayType) types.get(i);
String arg = convertParam(names.get(i), type.getComponentType());
args.append('*' + arg);
}
else {
args.append(convertParam(names.get(i), types.get(i)));
}
}
printer.print("(");
printer.print(args);
printer.print(")");
TypeMirror res = el.getReturnType();
if (res.getKind() != TypeKind.VOID) {
printer.print(" -> ");
String convertedType = convertResultType(res);
if (convertedType != null) {
printer.print(convertedType);
}
else {
printer.print(getTypeString(parent.el, res));
}
}
}
/**
* Gets the property name for this method if applicable
*
* @return the property name or null
*/
private String getPropertyName() {
String name = getName();
if (name.startsWith("get") || name.startsWith("set")) {
return Character.toLowerCase(name.charAt(3)) + name.substring(4);
}
if (name.startsWith("is")) {
return Character.toLowerCase(name.charAt(2)) + name.substring(3);
}
return null;
}
/**
* Gets a list of all the parameter types
*
* @return the list of parameter types
*/
private List<? extends TypeMirror> getParameterTypes() {
return ((ExecutableType) el.asType()).getParameterTypes();
}
/**
* Gets a list of all the Python safe parameter names
*
* @return the list of parameter names
*/
private List<String> getParameterNames() {
List<? extends VariableElement> params = el.getParameters();
List<String> names = new ArrayList<>(params.size());
for (VariableElement param : params) {
String name = sanitize(param.getSimpleName());
if (name.equals("self")) {
name = "self_";
}
names.add(name);
}
return names;
}
/**
* Gets the return type
*
* @return the return type
*/
private TypeMirror getReturnType() {
return el.getReturnType();
}
/**
* Converts the provided parameter type to a typing.Union of all the allowed types
*
* @param name the parameter name
* @param type the parameter type
* @return the parameter and its type
*/
private String convertParam(String name, TypeMirror type) {
String convertedType = convertParamType(type);
if (convertedType != null) {
return name + ": " + convertedType;
}
return name + ": " + getTypeString(parent.el, type);
}
/**
* Converts the provided parameter type to a typing.Union of all the allowed types
*
* @param type the parameter type
* @return the converted type
*/
private static String convertParamType(TypeMirror type) {
if (type.getKind().isPrimitive()) {
return switch (type.getKind()) {
case BOOLEAN -> "typing.Union[jpype.JBoolean, bool]";
case BYTE -> "typing.Union[jpype.JByte, int]";
case CHAR -> "typing.Union[jpype.JChar, int, str]";
case DOUBLE -> "typing.Union[jpype.JDouble, float]";
case FLOAT -> "typing.Union[jpype.JFloat, float]";
case INT -> "typing.Union[jpype.JInt, int]";
case LONG -> "typing.Union[jpype.JLong, int]";
case SHORT -> "typing.Union[jpype.JShort, int]";
default -> throw new RuntimeException("unexpected TypeKind " + type.getKind());
};
}
if (type instanceof DeclaredType dt) {
Element element = dt.asElement();
if (element instanceof QualifiedNameable nameable) {
return AUTO_CONVERSIONS.get(nameable.getQualifiedName().toString());
}
}
return null;
}
/**
* Helper for creating a Python property.<p/>
*
* This class only represents one part of a complete Python property.
*/
class PropertyMethod {
/**
* Gets the name for this property
*
* @return the property name
*/
String getName() {
return sanitize(getPropertyName());
}
/**
* Checks if this property is a getter
*
* @return true if this property is a getter
*/
boolean isGetter() {
return el.getReturnType().getKind() != TypeKind.VOID;
}
/**
* Checks if this property is a setter
*
* @return true if this property is a setter
*/
boolean isSetter() {
return el.getReturnType().getKind() == TypeKind.VOID;
}
/**
* Gets the type for this property
*
* @return the property type
*/
TypeMirror getType() {
TypeMirror type;
if (isGetter()) {
type = el.getReturnType();
}
else {
type = getParameterTypes().get(0);
}
try {
return doclet.getTypeUtils().unboxedType(type);
}
catch (IllegalArgumentException e) {
// not boxed
return type;
}
}
/**
* Checks if this property and the other provided property form a pair
*
* @param other the other property
* @return true if the two properties form a pair
*/
boolean isPair(PropertyMethod other) {
if (isGetter() && other.isGetter()) {
return false;
}
if (isSetter() && other.isSetter()) {
return false;
}
if (!getName().equals(other.getName())) {
return false;
}
return doclet.getTypeUtils().isSameType(getType(), other.getType());
}
}
@Override
public int compareTo(PythonTypeStubMethod other) {
return getName().compareTo(other.getName());
}
}
@@ -0,0 +1,30 @@
package ghidra.doclets.typestubs;
import java.io.PrintWriter;
import javax.lang.model.element.TypeElement;
/**
* {@link PythonTypeStubElement} for a nested type
*/
final class PythonTypeStubNestedType extends PythonTypeStubType {
// while it is possible to create a pseudo sub module to
// make static nested classes and enum values individually
// importable during type checking, it's not worth the effort
/**
* Creates a new {@link PythonTypeStubNestedType}
*
* @param pkg the package containing this type
* @param el the element for this type
*/
PythonTypeStubNestedType(PythonTypeStubPackage pkg, TypeElement el) {
super(pkg, el);
}
@Override
void process(PrintWriter printer, String indent) {
printClass(printer, indent);
}
}
@@ -0,0 +1,242 @@
package ghidra.doclets.typestubs;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import javax.lang.model.element.Element;
import javax.lang.model.element.PackageElement;
import javax.lang.model.element.TypeElement;
/**
* {@link PythonTypeStubElement} for a package<p/>
*
* This will process all visible classes, interfaces, handle necessary imports
* and create the __init__.pyi file.
*/
final class PythonTypeStubPackage extends PythonTypeStubElement<PackageElement> {
private String packageName;
private File path;
private List<PythonTypeStubType> types;
/**
* Creates a new {@link PythonTypeStubPackage}
*
* @param doclet the current doclet
* @param el the element for this package
*/
PythonTypeStubPackage(PythonTypeStubDoclet doclet, PackageElement el) {
super(doclet, el);
}
/**
* Gets a list of all the TypeVars needed by the types in this package
*
* @return a list of all the needed TypeVars
*/
List<String> getTypeVars() {
Set<String> typevars = new HashSet<>();
for (PythonTypeStubType type : getTypes()) {
typevars.addAll(type.getTypeVars());
}
List<String> res = new ArrayList<>(typevars);
res.sort(null);
return res;
}
/**
* Gets a collection of all the imported types needed by the types in this package
*
* @return a collection of all the imported types
*/
Collection<TypeElement> getImportedTypes() {
Set<TypeElement> imported = new HashSet<>();
for (PythonTypeStubType type : getTypes()) {
imported.addAll(type.getImportedTypes());
}
return imported;
}
/**
* Gets the Python safe, fully qualified name for this package
*
* @return the qualified package name
*/
String getPackageName() {
if (packageName == null) {
packageName = sanitizeQualifiedName(el.getQualifiedName().toString());
}
return packageName;
}
/**
* Processes this package and its contents to create a __init__.pyi file
*/
void process() {
doclet.addProcessedPackage(el);
getPath().mkdirs();
File stub = new File(path, "__init__.pyi");
try (PrintWriter printer = new PrintWriter(new FileWriter(stub))) {
process(printer, "");
}
catch (IOException e) {
e.printStackTrace();
}
}
/**
* Gets a list of all the types declared in this package
*
* @return a list of all specified types
*/
List<PythonTypeStubType> getTypes() {
// NOTE: do ALL SPECIFIED TYPES
// if it is not public, it will be decorated with @typing.type_check_only
// this prevents errors during typechecking from having a class with a base
// class that doesn't have public visibility
if (types != null) {
return types;
}
types = new ArrayList<>();
for (Element child : el.getEnclosedElements()) {
switch (child.getKind()) {
case CLASS:
case INTERFACE:
case ENUM:
case RECORD:
if (!doclet.isSpecified(child)) {
continue;
}
types.add(new PythonTypeStubType(this, (TypeElement) child));
break;
default:
break;
}
}
return types;
}
/**
* Process the contents of this package and write the results to the provided printer
*
* @param printer the printer to write to
* @param indent the current indentation
*/
private void process(PrintWriter printer, String indent) {
writeJavaDoc(printer, indent, "");
printer.println("import collections.abc");
printer.println("import datetime");
printer.println("import typing");
printer.println("from warnings import deprecated # type: ignore");
printer.println();
printer.println("import jpype # type: ignore");
printer.println("import jpype.protocol # type: ignore");
printer.println();
doclet.printImports(printer, getImportedPackages());
printer.println();
printer.println();
printTypeVars(printer);
Set<String> exports = new LinkedHashSet<>();
for (PythonTypeStubType type : getTypes()) {
processType(printer, indent, type);
exports.add('"' + type.getName() + '"');
}
printer.println();
// create the __all__ variable to prevent our imports and TypeVars from being
// imported when "from {getPackageName()} import *" is used
printer.print("__all__ = [");
printer.print(String.join(", ", exports));
printer.println("]");
}
/**
* Gets the output directory for this package
*
* @return the output directory
*/
private File getPath() {
if (path == null) {
String name = getPackageName();
int index = name.indexOf('.');
if (index != -1) {
name = name.substring(0, index) + "-stubs" + name.substring(index);
}
else {
name += "-stubs";
}
path = new File(doclet.getDestDir(), name.replace('.', '/'));
}
return path;
}
/**
* Gets a collection of all imported packages
*
* @return a collection of all imported packages
*/
private Collection<PackageElement> getImportedPackages() {
Set<PackageElement> packages = new HashSet<>();
for (TypeElement element : getImportedTypes()) {
if (isNestedType(element)) {
// don't import types declared in this file
continue;
}
PackageElement importedPkg = getPackage(element);
if (importedPkg == null || el.equals(importedPkg)) {
continue;
}
packages.add(importedPkg);
}
List<PackageElement> res = new ArrayList<>(packages);
res.sort(PythonTypeStubElement::compareQualifiedNameable);
return res;
}
/**
* Processes the provided type and write it to the provided printer
*
* @param printer the printer
* @param indent the current indentation
* @param type the type
*/
private void processType(PrintWriter printer, String indent, PythonTypeStubType type) {
type.process(printer, indent);
}
/**
* Checks if the provided type is a nested type
*
* @param element the type element to check
* @return true if the type is declared within another class
*/
private static boolean isNestedType(TypeElement element) {
return element.getEnclosingElement() instanceof TypeElement;
}
/**
* Prints all the typevars to the provided printer
*
* @param printer the printer
*/
private void printTypeVars(PrintWriter printer) {
List<String> allTypeVars = getTypeVars();
for (String generic : allTypeVars) {
printer.println(generic + " = typing.TypeVar(\"" + generic + "\")");
}
if (!allTypeVars.isEmpty()) {
printer.println();
printer.println();
}
}
}
@@ -0,0 +1,356 @@
package ghidra.doclets.typestubs;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import javax.lang.model.element.Element;
import com.sun.source.doctree.DocTree;
/**
* Helper class for converting an HTML table to reStructuredText
*/
final class RstTableBuilder {
// give each column enough padding to allow an alignment
private static final int COLUMN_PADDING = 2;
private final HtmlConverter docConverter;
private final Element el;
private Row columns = new Row();
private List<Row> rows = new ArrayList<>();
private Row currentRow = null;
private List<Integer> columnWidths = new ArrayList<>();
private String caption = null;
/**
* Creates a new {@link RstTableBuilder}
*
* @param docConverter the html converter
* @param el the element
*/
RstTableBuilder(HtmlConverter docConverter, Element el) {
this.docConverter = docConverter;
this.el = el;
}
/**
* Adds new row group to the table
*
* @param tree the html tree containing the row group
* @throws UnsupportedOperationException if any row in the group contains a nested row
*/
void addRowGroup(HtmlDocTree tree) {
switch (tree.getHtmlKind()) {
case THEAD:
if (tree.getBody().size() > 1) {
throw new UnsupportedOperationException("nested table rows are not supported");
}
case TBODY:
case TFOOT:
for (DocTree tag : tree.getBody()) {
if (!(tag instanceof HtmlDocTree)) {
continue;
}
addRow((HtmlDocTree) tag);
}
return;
default:
break;
}
}
/**
* Adds new row to the table
*
* @param tree the html tree containing the row
* @throws UnsupportedOperationException if the row contains a nested row
*/
void addRow(HtmlDocTree tree) {
if (currentRow == null) {
currentRow = columns;
}
else {
currentRow = new Row();
rows.add(currentRow);
}
boolean columnsDone = columns.size() > 0;
for (DocTree tag : tree.getBody()) {
if (!(tag instanceof HtmlDocTree)) {
continue;
}
HtmlDocTree html = (HtmlDocTree) tag;
String align = docConverter.getAttributes(el, html.getStartTag()).get("align");
switch (html.getHtmlKind()) {
case TH:
if (columnsDone) {
// vertical headers
// insert it as an entry so it at least comes out ok
addEntry(getBody(html), align);
}
else {
addColumn(getBody(html), align);
}
break;
case TD:
addEntry(getBody(html), align);
break;
case TR:
throw new UnsupportedOperationException("nested table rows are not supported");
default:
break;
}
}
}
/**
* Adds a caption to the table
*
* @param caption the caption
*/
void addCaption(String caption) {
if (!caption.isBlank()) {
this.caption = caption;
}
}
/**
* Builds the reStructuredText formatted table
*
* @return the reStructuredText table
*/
String build() {
StringBuilder builder = new StringBuilder();
builder.append('\n');
if (caption != null) {
int length = caption.length();
builder.repeat('^', length)
.append('\n');
builder.append(caption)
.append('\n')
.repeat('^', length)
.append('\n');
}
buildRowBorder(builder, '-');
columns.build(builder);
buildRowBorder(builder, '=');
for (Row row : rows) {
row.build(builder);
buildRowBorder(builder, '-');
}
return builder.toString();
}
/**
* Adds a column to the table
*
* @param value the column value
* @param align the column alignment
*/
private void addColumn(String value, String align) {
if (align == null) {
align = "CENTER";
}
addColumn(value, Alignment.valueOf(align.toUpperCase()));
}
/**
* Adds a column to the table
*
* @param value the column value
* @param align the column alignment
*/
private void addColumn(String value, Alignment align) {
int column = columns.size();
columns.addValue(value, align);
growColumn(value, column);
}
/**
* Adds an entry to the current row in the table
*
* @param value the entry value
* @param align the entry alignment
*/
private void addEntry(String value, String align) {
if (align == null) {
align = "LEFT";
}
addEntry(value, Alignment.valueOf(align.toUpperCase()));
}
/**
* Adds an entry to the current row in the table
*
* @param value the entry value
* @param align the entry alignment
*/
private void addEntry(String value, Alignment align) {
int column = currentRow.size();
currentRow.addValue(value, align);
growColumn(value, column);
}
/**
* Helper method to get the converted contents of an html tree
*
* @param tag the html
* @return the converted html
*/
private String getBody(HtmlDocTree tag) {
return docConverter.convertTree(el, tag.getBody());
}
/**
* Creates a row border with the provided character
*
* @param builder the string builder
* @param c the border character
*/
private void buildRowBorder(StringBuilder builder, char c) {
builder.append('+');
for (int width : columnWidths) {
builder.repeat(c, width)
.append('+');
}
builder.append('\n');
}
/**
* Computes the max line width for the provided multi-line text
*
* @param text the text
* @return the max line width
*/
private static int getLineWidth(String text) {
// value may be mutiple lines
return text.lines()
.map(String::stripLeading)
.mapToInt(String::length)
.max()
.getAsInt();
}
/**
* Grows the provided column appropriately for the newly added value
*
* @param value the newly added value
* @param column the column number
*/
private void growColumn(String value, int column) {
int length = !value.isEmpty() ? getLineWidth(value) + COLUMN_PADDING : COLUMN_PADDING;
if (column >= columnWidths.size()) {
columnWidths.add(length);
return;
}
if (columnWidths.get(column) < length) {
columnWidths.set(column, length);
}
}
/**
* Aligns the single line value according to the column width and alignment
*
* @param value the value to align
* @param columnWidth the column width
* @param align the alignment
* @return the aligned value
*/
private static String alignSingleLine(String value, int columnWidth, Alignment align) {
int length = value.length();
return switch (align) {
case LEFT -> value + " ".repeat(columnWidth - length);
case CENTER -> {
int left = (columnWidth - length) / 2;
int right = left;
if (left + right + length < columnWidth) {
right++;
}
yield " ".repeat(left) + value + " ".repeat(right);
}
case RIGHT -> " ".repeat(columnWidth - length) + value;
};
}
private static enum Alignment {
LEFT,
CENTER,
RIGHT
}
/**
* Helper class for modeling a table row
*/
private class Row {
int maxLines = 1;
List<List<String>> values = new ArrayList<>();
List<Alignment> alignments = new ArrayList<>();
/**
* Adds the value to the row
*
* @param value the value
* @param align the alignment
*/
void addValue(String value, Alignment align) {
List<String> lines = value.lines()
.map(String::stripLeading)
.collect(Collectors.toList());
if (lines.size() > maxLines) {
maxLines = lines.size();
}
values.add(lines);
alignments.add(align);
}
/**
* Gets the size of this row
*
* @return the row size
*/
int size() {
return values.size();
}
/**
* Appends this row to the provided string builder
*
* @param builder the string builder
*/
void build(StringBuilder builder) {
for (int i = 0; i < maxLines; i++) {
builder.append('|');
for (int j = 0; j < values.size(); j++) {
List<String> entry = values.get(j);
String value;
if (i >= entry.size()) {
value = " ".repeat(columnWidths.get(j));
}
else {
value = alignSingleLine(j, entry.get(i));
}
builder.append(value)
.append('|');
}
builder.append('\n');
}
}
/**
* Aligns the provided single line value according to the column and its alignent
*
* @param column the column number
* @param value the single line value
* @return the aligned value
*/
String alignSingleLine(int column, String value) {
int columnLength = columnWidths.get(column);
return RstTableBuilder.alignSingleLine(value, columnLength, alignments.get(column));
}
}
}