Mirrors_Framework/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2026-05-21 23:52:28 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

GP-1830 BSim migration to gson use. Corrected various bugs with BSim

Browse Source 
elasticsearch use.
This commit is contained in:
ghidra1
2024-12-12 13:51:28 -05:00
parent 237d0445b5
commit 593fd98e0d
18 changed files with 769 additions and 1275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Ghidra/Features/BSim/Module.manifest
-1
View File
@@ -1,7 +1,6 @@
##MODULE IP: Oxygen Icons - LGPL 3.0
MODULE FILE LICENSE: postgresql-15.3.tar.gz Postgresql License
MODULE FILE LICENSE: lib/postgresql-42.6.2.jar PostgresqlJDBC License
MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0
MODULE FILE LICENSE: lib/commons-dbcp2-2.9.0.jar Apache License 2.0
MODULE FILE LICENSE: lib/commons-pool2-2.11.1.jar Apache License 2.0
MODULE FILE LICENSE: lib/commons-logging-1.2.jar Apache License 2.0
Ghidra/Features/BSim/build.gradle
-1
View File
@@ -33,7 +33,6 @@ dependencies {
api project(":CodeCompare")
api "org.postgresql:postgresql:42.6.2"
api "com.googlecode.json-simple:json-simple:1.1.1"
api "org.apache.commons:commons-dbcp2:2.9.0"
api "org.apache.commons:commons-pool2:2.11.1"
api "commons-logging:commons-logging:1.2"
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/ExecutableNameBSimFilterType.java
+5 -7
View File
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters;
import java.sql.SQLException;
import org.json.simple.JSONObject;
import ghidra.features.bsim.query.client.IDSQLResolution;
import ghidra.features.bsim.query.client.SQLEffects;
import ghidra.features.bsim.query.description.ExecutableRecord;
@@ -38,7 +36,7 @@ public class ExecutableNameBSimFilterType extends BSimFilterType {
@Override
public void gatherSQLEffect(SQLEffects effect, FilterAtom atom, IDSQLResolution resolution)
throws SQLException {
throws SQLException {
effect.setExeTable();
StringBuilder buf = new StringBuilder();
buf.append("exetable.name_exec = '").append(atom.value).append('\'');
@@ -47,10 +45,10 @@ public class ExecutableNameBSimFilterType extends BSimFilterType {
@Override
public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom,
IDElasticResolution resolution) throws ElasticException {
IDElasticResolution resolution) throws ElasticException {
StringBuilder buffer = new StringBuilder();
buffer.append("\"filter\": { \"term\": { \"name_exec\": \"");
buffer.append(JSONObject.escape(atom.value));
buffer.append(ElasticDatabase.escape(atom.value));
buffer.append("\" } } ");
effect.addStandalone(this, buffer.toString());
}
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/NotExecutableNameBSimFilterType.java
+5 -7
View File
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,8 +17,6 @@ package ghidra.features.bsim.gui.filters;
import java.sql.SQLException;
import org.json.simple.JSONObject;
import ghidra.features.bsim.query.client.IDSQLResolution;
import ghidra.features.bsim.query.client.SQLEffects;
import ghidra.features.bsim.query.description.ExecutableRecord;
@@ -37,7 +35,7 @@ public class NotExecutableNameBSimFilterType extends BSimFilterType {
@Override
public void gatherSQLEffect(SQLEffects effect, FilterAtom atom, IDSQLResolution resolution)
throws SQLException {
throws SQLException {
effect.setExeTable();
StringBuilder buf = new StringBuilder();
buf.append("exetable.name_exec != '").append(atom.value).append('\'');
@@ -46,10 +44,10 @@ public class NotExecutableNameBSimFilterType extends BSimFilterType {
@Override
public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom,
IDElasticResolution resolution) throws ElasticException {
IDElasticResolution resolution) throws ElasticException {
StringBuilder buffer = new StringBuilder();
buffer.append("\"must_not\": { \"term\": { \"name_exec\": \"");
buffer.append(JSONObject.escape(atom.value));
buffer.append(ElasticDatabase.escape(atom.value));
buffer.append("\" } } ");
effect.addStandalone(this, buffer.toString());
}
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/gui/filters/PathStartsBSimFilterType.java
+6 -7
View File
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,7 +35,7 @@ public class PathStartsBSimFilterType extends BSimFilterType {
@Override
public void gatherSQLEffect(SQLEffects effect, FilterAtom atom, IDSQLResolution resolution)
throws SQLException {
throws SQLException {
if (atom.value.length() > 0) {
effect.setExeTable();
effect.setPathTable();
@@ -47,11 +47,10 @@ public class PathStartsBSimFilterType extends BSimFilterType {
@Override
public void gatherElasticEffect(ElasticEffects effect, FilterAtom atom,
IDElasticResolution resolution) throws ElasticException {
effect.addDocValue("String path = doc['path'].value; ");
IDElasticResolution resolution) throws ElasticException {
effect.addDocValue("String path = doc['path'].size() == 0 ? null : doc['path'].value; ");
String argName = effect.assignArgument();
effect.addScriptElement(this,
"(path != null) && path.startsWith(params." + argName + ')');
effect.addScriptElement(this, "(path != null) && path.startsWith(params." + argName + ')');
effect.addParam(argName, atom.value);
}
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/BSimPostgresDBConnectionManager.java
+3 -1
View File
@@ -284,7 +284,9 @@ public class BSimPostgresDBConnectionManager {
String loginError = null;
serverInfo.setUserInfo(bds);
if (bds.getPassword() == null) {
serverInfo.setUserInfo(bds);
}
connectionType = serverInfo.hasPassword() ? ConnectionType.SSL_Password_Authentication
: ConnectionType.SSL_No_Authentication;
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/AbstractSQLFunctionDatabase.java
+37 -4
View File
@@ -186,7 +186,7 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
* @throws SQLException if error occurs obtaining connection
*/
protected Connection initConnection() throws SQLException {
if (db == null) {
if (db == null || db.isClosed()) {
db = ds.getConnection();
}
return db;
@@ -435,6 +435,12 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
}
}
/**
* Drop this database
* @throws SQLException if a database error occured
*/
abstract protected void dropDatabase() throws SQLException;
protected void setConnectionOnTables(Connection db) {
weightTable.setConnection(db);
@@ -1201,8 +1207,8 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
}
else if (msg.contains("authentication failed") ||
msg.contains("requires a valid client certificate")) {
lasterror =
new BSimError(ErrorCategory.Authentication, "Could not authenticate with database");
lasterror = new BSimError(ErrorCategory.Authentication,
"Could not authenticate with database");
}
else if (msg.contains("does not exist") && !msg.contains(" role ")) {
lasterror = new BSimError(ErrorCategory.Nodatabase, cause.getMessage());
@@ -1572,6 +1578,9 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
else if (query instanceof QueryExeCount q) {
fdbQueryExeCount(q);
}
else if (query instanceof DropDatabase q) {
fdbDatabaseDrop(q);
}
else if (query instanceof CreateDatabase q) {
fdbDatabaseCreate(q);
}
@@ -1622,7 +1631,8 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
lasterror = null;
try {
if (!(query instanceof CreateDatabase) && !initialize()) {
if (!(query instanceof CreateDatabase) && !(query instanceof DropDatabase) &&
!initialize()) {
lasterror = new BSimError(ErrorCategory.Nodatabase, "The database does not exist");
return null;
}
@@ -2087,6 +2097,29 @@ public abstract class AbstractSQLFunctionDatabase<VF extends LSHVectorFactory>
}
}
private void fdbDatabaseDrop(DropDatabase query) throws LSHException {
ResponseDropDatabase response = query.getResponse();
if (query.databaseName == null) {
throw new LSHException("Missing databaseName for drop database");
}
if (!query.databaseName.equals(ds.getServerInfo().getDBName())) {
throw new UnsupportedOperationException("drop database name must match");
}
response.dropSuccessful = true; // Response parameters assuming success
response.errorMessage = null;
try {
dropDatabase();
}
catch (SQLException e) {
String msg = e.getMessage();
if (msg.indexOf("database \"" + query.databaseName + "\" does not exist") > 0) {
return; // missing database
}
response.dropSuccessful = false;
response.errorMessage = e.getMessage();
}
}
/**
* Entry point for the CreateDatabase command
* @param query the query to execute
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/client/PostgresFunctionDatabase.java
+59 -3
View File
@@ -31,6 +31,7 @@ import ghidra.features.bsim.query.client.tables.CachedStatement;
import ghidra.features.bsim.query.client.tables.SQLStringTable;
import ghidra.features.bsim.query.description.*;
import ghidra.features.bsim.query.protocol.*;
import ghidra.util.Msg;
/**
* Defines the BSim {@link FunctionDatabase} backed by a PostgreSQL database.
@@ -200,9 +201,6 @@ public final class PostgresFunctionDatabase
st.executeUpdate(createdbstring);
postgresDs.initializeFrom(defaultDs);
}
finally {
defaultDs.dispose();
}
}
@Override
@@ -244,6 +242,64 @@ public final class PostgresFunctionDatabase
}
}
@Override
protected void dropDatabase() throws SQLException {
if (getStatus() == Status.Busy || postgresDs.getActiveConnections() != 0) {
throw new SQLException("database in use");
}
BSimServerInfo serverInfo = postgresDs.getServerInfo();
BSimServerInfo defaultServerInfo =
new BSimServerInfo(DBType.postgres, serverInfo.getUserInfo(),
serverInfo.getServerName(), serverInfo.getPort(), DEFAULT_DATABASE_NAME);
BSimPostgresDataSource defaultDs =
BSimPostgresDBConnectionManager.getDataSource(defaultServerInfo);
if (getStatus() == Status.Ready) {
defaultDs.initializeFrom(postgresDs);
}
close(); // close this instance
try (Connection defaultDb = defaultDs.getConnection();
Statement defaultSt = defaultDb.createStatement()) {
try (ResultSet rs = defaultSt.executeQuery(
"SELECT 1 FROM pg_database WHERE datname='" + serverInfo.getDBName() + "'")) {
if (!rs.next()) {
return; // database does not exist
}
}
// Connect to database and examine schema
HashSet<String> tableNames = new HashSet<>();
postgresDs.initializeFrom(defaultDs);
try (Connection c = initConnection(); Statement st = c.createStatement()) {
try (ResultSet rs = st.executeQuery(
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) {
while (rs.next()) {
tableNames.add(rs.getString(1));
}
}
}
// Spot check for a few BSim table names that always exist
if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") ||
!tableNames.contains("weighttable")) {
throw new SQLException("attempted to drop non-BSim database");
}
postgresDs.dispose(); // disconnect before dropping database
Msg.info(this, "Dropping BSim postgresql database: " + serverInfo);
defaultSt.executeUpdate("DROP DATABASE \"" + serverInfo.getDBName() + '"');
}
finally {
// ensure
postgresDs.initializeFrom(defaultDs);
}
}
/**
*
* @throws SQLException if there is a problem creating or executing the query
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticConnection.java
+124 -50
View File
@@ -19,9 +19,9 @@ import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import com.google.gson.*;
import ghidra.util.Msg;
public class ElasticConnection {
public static final String POST = "POST";
@@ -38,24 +38,51 @@ public class ElasticConnection {
httpURLbase = url + '/' + repo + '_';
}
public void close() {
// nothing to do - http connections do not persist
}
public boolean lastRequestSuccessful() {
return (lastResponseCode >= 200) && (lastResponseCode < 300);
}
/**
* Assuming the writer has been closed and connection.getResponseCode() is called
* placing the value in lastResponseCode, read the response and parse into a JSONObject
* @return the JSONObject
* @throws IOException for problems with the socket
* @throws ParseException for JSON parse errors
* Get String held by a JsonElement, allowing for a null object.
* @param element is the JsonElement or null
* @return the underlying String or null
*/
private JSONObject grabResponse(HttpURLConnection connection)
throws IOException, ParseException {
JSONParser parser = new JSONParser();
static String convertToString(JsonElement element) {
if (isNull(element)) {
return null;
}
return element.getAsString();
}
/**
* Get String held by a JsonElement, allowing for a null object.
* @param element is the JsonElement or null
* @param defaultStr default string to be returned if element or string is null
* @return the underlying String or defaultStr if null
*/
static String convertToString(JsonElement element, String defaultStr) {
String str = convertToString(element);
return str != null ? str : defaultStr;
}
/**
* Check element for null value
* @param element json element
* @return true if null else false
*/
static boolean isNull(JsonElement element) {
return (element == null || element instanceof JsonNull);
}
/**
* Assuming the writer has been closed and connection.getResponseCode() is called
* placing the value in lastResponseCode, read the response and parse into a JsonObject
* @return the JsonObject
* @throws IOException for problems with the socket
* @throws JsonParseException for JSON parse errors
*/
private JsonObject grabResponse(HttpURLConnection connection)
throws IOException, JsonParseException {
InputStream in;
if (lastRequestSuccessful()) {
in = connection.getInputStream();
@@ -68,7 +95,7 @@ public class ElasticConnection {
throw new IOException(connection.getResponseMessage());
}
Reader reader = new InputStreamReader(in);
JSONObject jsonObject = (JSONObject) parser.parse(reader);
JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject();
return jsonObject;
}
@@ -78,39 +105,86 @@ public class ElasticConnection {
* @param resp is the parsed error document
* @return the exception String
*/
private String parseErrorJSON(JSONObject resp) {
private static String parseErrorJSON(JsonObject resp) {
Object errorObj = resp.get("error");
if (errorObj == null) {
return "Unknown error format";
}
if (errorObj instanceof String) {
return (String) errorObj;
}
if (!(errorObj instanceof JSONObject)) {
if (!(errorObj instanceof JsonObject err)) {
return "Unknown error format";
}
JSONObject jsonObj = (JSONObject) errorObj;
String typeString = (String) jsonObj.get("type");
String reasonString = (String) jsonObj.get("reason");
if (typeString == null) {
typeString = "Unknown Error";
}
if (reasonString == null) {
reasonString = "Unknown reason";
String typeString = convertToString(err.get("type"), "Unknown Error");
if (typeString.endsWith("_exception")) {
// Log elastic exception root cause to assist debug
String errorDetail = parseErrorCause(err);
if (errorDetail.length() != 0) {
Msg.error(ElasticConnection.class, "Elasticsearch exception: " + errorDetail);
}
}
String reasonString = convertToString(err.get("reason"), "Unknown Reason");
return typeString + " : " + reasonString;
}
private static StringBuilder conditionalNewLine(StringBuilder buf) {
if (!buf.isEmpty()) {
buf.append("\n");
}
return buf;
}
private static String parseErrorCause(JsonObject error) {
StringBuilder buf = new StringBuilder();
JsonElement reason = error.get("reason");
String typeString = convertToString(error.get("type"));
if (typeString != null) {
String reasonString = convertToString(reason); // "reason" is string when "type" is present
String errorStr = typeString + " : " + reasonString;
conditionalNewLine(buf).append(errorStr);
}
JsonElement scriptStack = error.get("script_stack");
if (scriptStack instanceof JsonArray scriptStackArray) {
scriptStackArray
.forEach(e -> conditionalNewLine(buf).append(" ").append(convertToString(e)));
}
JsonElement causedBy = error.get("caused_by");
if (causedBy instanceof JsonObject causedByObject) {
conditionalNewLine(buf).append(" ").append(parseErrorCause(causedByObject));
}
JsonElement failedShards = error.get("failed_shards");
if (failedShards instanceof JsonArray failedShardsArray) {
for (JsonElement failedShardElement : failedShardsArray) {
JsonObject failedShard = (JsonObject) failedShardElement;
String indexStr = convertToString(failedShard.get("index"));
conditionalNewLine(buf).append(" Failed shard index: ").append(indexStr);
conditionalNewLine(buf).append(" ").append(parseErrorCause(failedShard));
}
}
if (reason instanceof JsonObject reasonObject) {
conditionalNewLine(buf).append(parseErrorCause(reasonObject));
}
return buf.toString();
}
/**
* Send a raw request to the server that is not specific to the repository.
* Intended for general configuration or security commands
* @param command is the type of command
* @param path is the specific URL path receiving the command
* @param body is JSON document describing the command
* @return the response as parsed JSONObject
* @return the response as parsed JsonObject
* @throws ElasticException for any problems with the connection
*/
public JSONObject executeRawStatement(String command, String path, String body)
public JsonObject executeRawStatement(String command, String path, String body)
throws ElasticException {
HttpURLConnection connection = null;
try {
@@ -123,7 +197,7 @@ public class ElasticConnection {
writer.write(body);
}
lastResponseCode = connection.getResponseCode();
JSONObject resp = grabResponse(connection);
JsonObject resp = grabResponse(connection);
if (!lastRequestSuccessful()) {
throw new ElasticException(parseErrorJSON(resp));
}
@@ -132,7 +206,7 @@ public class ElasticConnection {
catch (IOException e) {
throw new ElasticException("Error sending request: " + e.getMessage());
}
catch (ParseException e) {
catch (JsonParseException e) {
throw new ElasticException("Error parsing response: " + e.getMessage());
}
finally {
@@ -163,7 +237,7 @@ public class ElasticConnection {
writer.write(body);
}
lastResponseCode = connection.getResponseCode();
JSONObject resp = grabResponse(connection);
JsonObject resp = grabResponse(connection);
if (!lastRequestSuccessful()) {
throw new ElasticException(parseErrorJSON(resp));
}
@@ -171,7 +245,7 @@ public class ElasticConnection {
catch (IOException e) {
throw new ElasticException("Error sending request: " + e.getMessage());
}
catch (ParseException e) {
catch (JsonParseException e) {
throw new ElasticException("Error parsing response: " + e.getMessage());
}
finally {
@@ -186,10 +260,10 @@ public class ElasticConnection {
* @param command is the type of command
* @param path is the overarching index/type/<command>
* @param body is JSON document describing the request
* @return the parsed response as a JSONObject
* @return the parsed response as a JsonObject
* @throws ElasticException for any problems with the connection
*/
public JSONObject executeStatement(String command, String path, String body)
public JsonObject executeStatement(String command, String path, String body)
throws ElasticException {
HttpURLConnection connection = null;
try {
@@ -202,7 +276,7 @@ public class ElasticConnection {
writer.write(body);
}
lastResponseCode = connection.getResponseCode();
JSONObject resp = grabResponse(connection);
JsonObject resp = grabResponse(connection);
if (!lastRequestSuccessful()) {
throw new ElasticException(parseErrorJSON(resp));
}
@@ -211,7 +285,7 @@ public class ElasticConnection {
catch (IOException e) {
throw new ElasticException("Error sending request: " + e.getMessage());
}
catch (ParseException e) {
catch (JsonParseException e) {
throw new ElasticException("Error parsing response: " + e.getMessage());
}
finally {
@@ -227,10 +301,10 @@ public class ElasticConnection {
* @param command is the type of command
* @param path is the overarching index/type/<command>
* @param body is JSON document describing the request
* @return the parsed response as a JSONObject
* @return the parsed response as a JsonObject
* @throws ElasticException for any problems with the connection
*/
public JSONObject executeStatementExpectFailure(String command, String path, String body)
public JsonObject executeStatementExpectFailure(String command, String path, String body)
throws ElasticException {
HttpURLConnection connection = null;
try {
@@ -243,13 +317,13 @@ public class ElasticConnection {
writer.write(body);
}
lastResponseCode = connection.getResponseCode();
JSONObject resp = grabResponse(connection);
JsonObject resp = grabResponse(connection);
return resp;
}
catch (IOException e) {
throw new ElasticException("Error sending request: " + e.getMessage());
}
catch (ParseException e) {
catch (JsonParseException e) {
throw new ElasticException("Error parsing response: " + e.getMessage());
}
finally {
@@ -264,10 +338,10 @@ public class ElasticConnection {
* and is structured slightly differently from other commands.
* @param path is the specific URL path receiving the bulk command
* @param body is structured list of JSON commands and source
* @return the response as parsed JSONObject
* @return the response as parsed JsonObject
* @throws ElasticException for any problems with the connection
*/
public JSONObject executeBulk(String path, String body) throws ElasticException {
public JsonObject executeBulk(String path, String body) throws ElasticException {
HttpURLConnection connection = null;
try {
URL httpURL = new URL(hostURL + path);
@@ -279,7 +353,7 @@ public class ElasticConnection {
writer.write(body);
}
lastResponseCode = connection.getResponseCode();
JSONObject resp = grabResponse(connection);
JsonObject resp = grabResponse(connection);
if (!lastRequestSuccessful()) {
throw new ElasticException(parseErrorJSON(resp));
}
@@ -288,7 +362,7 @@ public class ElasticConnection {
catch (IOException e) {
throw new ElasticException("Error sending request: " + e.getMessage());
}
catch (ParseException e) {
catch (JsonParseException e) {
throw new ElasticException("Error parsing response: " + e.getMessage());
}
finally {
@@ -298,7 +372,7 @@ public class ElasticConnection {
}
}
public JSONObject executeURIOnly(String command, String path) throws ElasticException {
public JsonObject executeURIOnly(String command, String path) throws ElasticException {
HttpURLConnection connection = null;
try {
URL httpURL = new URL(httpURLbase + path);
@@ -306,7 +380,7 @@ public class ElasticConnection {
connection.setRequestMethod(command);
connection.setDoOutput(true);
lastResponseCode = connection.getResponseCode();
JSONObject resp = grabResponse(connection);
JsonObject resp = grabResponse(connection);
if (!lastRequestSuccessful()) {
throw new ElasticException(parseErrorJSON(resp));
}
@@ -315,7 +389,7 @@ public class ElasticConnection {
catch (IOException e) {
throw new ElasticException("Error sending request: " + e.getMessage());
}
catch (ParseException e) {
catch (JsonParseException e) {
throw new ElasticException("Error parsing response: " + e.getMessage());
}
finally {
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/elastic/ElasticDatabase.java
+337 -245
View File
File diff suppressed because it is too large Load Diff
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/BSimH2FileDBConnectionManager.java
+7 -1
View File
@@ -172,7 +172,7 @@ public class BSimH2FileDBConnectionManager {
dispose();
if (dbf.isFile()) {
if (!dbf.isFile()) {
return true;
}
@@ -286,6 +286,9 @@ public class BSimH2FileDBConnectionManager {
public synchronized Connection getConnection() throws SQLException {
if (successfulConnection) {
if (bds.isClosed()) {
bds.restart();
}
return bds.getConnection();
}
@@ -317,6 +320,9 @@ public class BSimH2FileDBConnectionManager {
* @throws SQLException if connection or authentication error occurs
*/
private Connection connect() throws SQLException {
if (bds.isClosed()) {
bds.restart();
}
Connection c = bds.getConnection();
successfulConnection = true;
return c;
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/file/H2FileFunctionDatabase.java
+67 -22
View File
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,13 +22,16 @@ import java.util.*;
import generic.concurrent.*;
import generic.lsh.vector.LSHVector;
import generic.lsh.vector.VectorCompare;
import ghidra.features.bsim.query.BSimServerInfo;
import ghidra.features.bsim.query.LSHException;
import ghidra.features.bsim.query.*;
import ghidra.features.bsim.query.BSimPostgresDBConnectionManager.BSimPostgresDataSource;
import ghidra.features.bsim.query.BSimServerInfo.DBType;
import ghidra.features.bsim.query.FunctionDatabase.Status;
import ghidra.features.bsim.query.client.*;
import ghidra.features.bsim.query.description.*;
import ghidra.features.bsim.query.elastic.Base64VectorFactory;
import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource;
import ghidra.features.bsim.query.protocol.*;
import ghidra.util.Msg;
import ghidra.util.task.TaskMonitor;
public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64VectorFactory> {
@@ -120,6 +123,48 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
}
}
@Override
protected void dropDatabase() throws SQLException {
if (getStatus() == Status.Busy || fileDs.getActiveConnections() != 0) {
throw new SQLException("database in use");
}
close(); // close this instance
if (!fileDs.exists()) {
// ignore request and return
return;
}
// Connect to database and examine schema
HashSet<String> tableNames = new HashSet<>();
try (Connection c = initConnection(); Statement st = c.createStatement()) {
try (ResultSet rs = st.executeQuery(
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name")) {
while (rs.next()) {
tableNames.add(rs.getString(1));
}
}
}
// Spot check for a few BSim table names that always exist
if (!tableNames.contains("keyvaluetable") || !tableNames.contains("desctable") ||
!tableNames.contains("weighttable")) {
throw new SQLException("attempted to drop non-BSim database");
}
fileDs.dispose(); // disconnect before deleting database
BSimServerInfo serverInfo = fileDs.getServerInfo();
if (!fileDs.delete()) {
throw new SQLException("failed to delete H2-file database: " + serverInfo);
}
Msg.info(this, "Deleted BSim H2-file database: " + serverInfo);
}
/**
* Create vector map which maps vector ID to {@link VectorStoreEntry}
* @return vector map
@@ -136,7 +181,7 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
@Override
public QueryResponseRecord doQuery(BSimQuery<?> query, Connection c)
throws SQLException, LSHException, DatabaseNonFatalException {
throws SQLException, LSHException, DatabaseNonFatalException {
if (query instanceof PrewarmRequest preWarmRequest) {
preWarmRequest.buildResponseTemplate();
@@ -174,8 +219,8 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
}
@Override
protected int queryNearestVector(List<VectorResult> resultset, LSHVector vec,
double simthresh, double sigthresh, int max) throws SQLException {
protected int queryNearestVector(List<VectorResult> resultset, LSHVector vec, double simthresh,
double sigthresh, int max) throws SQLException {
VectorCompare comp;
List<VectorResult> resultsToSort = new ArrayList<>();
for (VectorStoreEntry entry : vectorStore) {
@@ -192,7 +237,7 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
continue;
}
resultsToSort
.add(new VectorResult(entry.id(), entry.count(), cosine, sig, entry.vec()));
.add(new VectorResult(entry.id(), entry.count(), cosine, sig, entry.vec()));
}
resultsToSort.sort((r1, r2) -> Double.compare(r2.sim, r1.sim));
int maxResults = Math.min(max, resultsToSort.size());
@@ -220,19 +265,19 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
new ConcurrentQBuilder<>();
ConcurrentQ<FunctionDescription, SimilarityVectorResult> evalQ =
evalBuilder.setThreadPool(threadPool)
.setCollectResults(true)
.setMonitor(TaskMonitor.DUMMY)
.build((fd, m) -> {
List<VectorResult> resultset = new ArrayList<>();
queryNearestVector(resultset, fd.getSignatureRecord().getLSHVector(),
.setCollectResults(true)
.setMonitor(TaskMonitor.DUMMY)
.build((fd, m) -> {
List<VectorResult> resultset = new ArrayList<>();
queryNearestVector(resultset, fd.getSignatureRecord().getLSHVector(),
query.thresh, query.signifthresh, vectormax);
if (resultset.isEmpty()) {
return null;
}
SimilarityVectorResult simres = new SimilarityVectorResult(fd);
simres.addNotes(resultset);
return simres;
});
if (resultset.isEmpty()) {
return null;
}
SimilarityVectorResult simres = new SimilarityVectorResult(fd);
simres.addNotes(resultset);
return simres;
});
evalQ.add(toQuery);
try {
@@ -257,8 +302,8 @@ public class H2FileFunctionDatabase extends AbstractSQLFunctionDatabase<Base64Ve
@Override
public int queryFunctions(QueryNearest query, BSimSqlClause filter, ResponseNearest response,
DescriptionManager descMgr, Iterator<FunctionDescription> iter)
throws SQLException, LSHException {
DescriptionManager descMgr, Iterator<FunctionDescription> iter)
throws SQLException, LSHException {
//TODO: is this what the method should return
//TODO: why is iter an argument? Why not just use query.manage.listAllFunctions()
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/DropDatabase.java
+53
View File
@@ -0,0 +1,53 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.query.protocol;
import java.io.IOException;
import java.io.Writer;
import generic.lsh.vector.LSHVectorFactory;
import ghidra.util.xml.XmlUtilities;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
public class DropDatabase extends BSimQuery<ResponseDropDatabase> {
public String databaseName;
public ResponseDropDatabase dropResponse;
public DropDatabase() {
super("dropdatabase");
}
@Override
public void buildResponseTemplate() {
if (response == null)
response = dropResponse = new ResponseDropDatabase();
}
@Override
public void saveXml(Writer fwrite) throws IOException {
fwrite.append('<').append(XmlUtilities.escapeElementEntities(name));
fwrite.append(" dbname=\"").append(databaseName).append("\" />\n");
}
@Override
public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory) {
XmlElement el = parser.start(name);
databaseName = XmlUtilities.unEscapeElementEntities(el.getAttribute("dbname"));
parser.end();
}
}
Ghidra/Features/BSim/src/main/java/ghidra/features/bsim/query/protocol/ResponseDropDatabase.java
+66
View File
@@ -0,0 +1,66 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.query.protocol;
import java.io.IOException;
import java.io.Writer;
import generic.lsh.vector.LSHVectorFactory;
import ghidra.features.bsim.query.LSHException;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
/**
* Response of server indicating whether a password change request ({@link PasswordChange}) succeeded
*/
public class ResponseDropDatabase extends QueryResponseRecord {
public boolean operationSupported; // true if the back-end supports this operation
public boolean dropSuccessful; // true if drop was successful
public String errorMessage; // Error message if change was not successful
public ResponseDropDatabase() {
super("responsedropdatabase");
operationSupported = true;
dropSuccessful = false;
errorMessage = null;
}
@Override
public void saveXml(Writer fwrite) throws IOException {
fwrite.append('<').append(name);
fwrite.append(" success=\"");
SpecXmlUtils.encodeBoolean(dropSuccessful);
fwrite.append("\">");
if (errorMessage != null) {
SpecXmlUtils.xmlEscapeWriter(fwrite, errorMessage);
}
fwrite.append("</").append(name).append(">\n");
}
@Override
public void restoreXml(XmlPullParser parser, LSHVectorFactory vectorFactory)
throws LSHException {
XmlElement el = parser.start(name);
dropSuccessful = SpecXmlUtils.decodeBoolean(el.getAttribute("success"));
errorMessage = parser.end().getText();
if (errorMessage != null && errorMessage.length() == 0) {
errorMessage = null;
}
}
}
Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/file/BSimH2DatabaseManagerTest.java
-1
View File
@@ -23,7 +23,6 @@ import java.util.*;
import org.junit.*;
import ghidra.features.bsim.query.*;
import ghidra.features.bsim.query.BSimServerInfo.DBType;
import ghidra.features.bsim.query.FunctionDatabase.BSimError;
import ghidra.features.bsim.query.description.DatabaseInformation;
import ghidra.features.bsim.query.file.BSimH2FileDBConnectionManager.BSimH2FileDataSource;
Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTest.java
-732
View File
File diff suppressed because it is too large Load Diff
Ghidra/Features/BSim/src/test.slow/java/ghidra/features/bsim/query/test/BSimServerTestUtil.java
-192
View File
@@ -1,192 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.features.bsim.query.test;
import java.io.*;
import ghidra.features.bsim.query.BSimControlLaunchable;
import ghidra.util.MD5Utilities;
import utilities.util.FileUtilities;
/**
* The TEST_DIRECTORY String should be changed to point to a directory that will
* hold data for the server and for the tests. To start, this directory should contain
* a subdirectory "raw", and within this subdirectory should be the following 3 specific binary
* executables:
* libreadline.so.7.0
* libhistory.so.7.0
* bash
*
* all pulled from Ubuntu 18.04.5.
*/
public class BSimServerTestUtil {
private static final String HOST_URL = "postgresql://localhost";
// private static final String HOST_URL = "https://localhost:9200";
// private static final String HOST_URL = "file:///tmp/bsimtest/db";
private static final String TEST_DIRECTORY = "/tmp/bsimtest";
public static final String REPO_NAME = "repo";
public static final String LIBHISTORY_MD5 = "0a860a716d5bec97c64db652549b72fd";
public static final String LIBREADLINE_MD5 = "71b5761b43b840eb88d053790deaf77c";
public static final String BASH_MD5 = "557c0271e30cf474e0f46f93721fd1ba";
public String repoName;
public String bsimURLString = HOST_URL + '/' + REPO_NAME;
public String testDir;
public String ghidraDir;
public String projectDir;
public String rawDir;
public String xmlDir;
public String serverDir;
public String serverTouchDir;
public boolean isElasticSearch;
public boolean isH2Database;
public BSimServerTestUtil() {
testDir = TEST_DIRECTORY;
ghidraDir = TEST_DIRECTORY + "/ghidra";
repoName = REPO_NAME;
projectDir = testDir + "/project";
rawDir = testDir + "/raw";
xmlDir = testDir + "/xml";
serverDir = testDir + "/db";
serverTouchDir = testDir + "/servertouch";
isElasticSearch = HOST_URL.startsWith("http") || HOST_URL.startsWith("elastic");
isH2Database = HOST_URL.startsWith("file");
}
public void verifyDirectories() throws FileNotFoundException {
File dir0 = new File(testDir);
if (!dir0.exists()) {
throw new FileNotFoundException("Could not find test directory");
}
File dir1 = new File(projectDir);
if (!dir1.exists()) {
if (!dir1.mkdir()) {
throw new FileNotFoundException("Could not create project directory");
}
}
File dir2 = new File(xmlDir);
if (!dir2.exists()) {
if (!dir2.mkdir()) {
throw new FileNotFoundException("Could not create xml directory");
}
}
File dir3 = new File(ghidraDir);
if (!dir3.exists()) {
if (!dir3.mkdir()) {
throw new FileNotFoundException("Could not create ghidra directory");
}
}
}
public void verifyRaw() throws IOException {
File rawDirectory = new File(rawDir);
if (!rawDirectory.exists()) {
throw new FileNotFoundException(rawDir);
}
if (!rawDirectory.isDirectory()) {
throw new FileNotFoundException("/raw is not a directory");
}
String[] list = rawDirectory.list();
boolean readlinePresent = false;
boolean historyPresent = false;
boolean bashPresent = false;
for (String element : list) {
File lib = new File(rawDirectory, element);
if (element.equals("libreadline.so.7.0")) {
String md5 = MD5Utilities.getMD5Hash(lib);
if (md5.equals(LIBREADLINE_MD5)) {
readlinePresent = true;
}
else {
throw new FileNotFoundException("libreadline.so.7.0 md5 does not match");
}
}
else if (element.equals("libhistory.so.7.0")) {
String md5 = MD5Utilities.getMD5Hash(lib);
if (md5.equals(LIBHISTORY_MD5)) {
historyPresent = true;
}
else {
throw new FileNotFoundException("libhistory.so.7.0 md5 does not match");
}
}
else if (element.equals("bash")) {
String md5 = MD5Utilities.getMD5Hash(lib);
if (md5.equals(BASH_MD5)) {
bashPresent = true;
}
else {
throw new FileNotFoundException("bash md5 does not match");
}
}
}
if (!readlinePresent) {
throw new FileNotFoundException("Missing libreadline.so.7.0");
}
if (!historyPresent) {
throw new FileNotFoundException("Missing libhistory.so.7.0");
}
if (!bashPresent) {
throw new FileNotFoundException("Missing bash");
}
}
public void startServer() throws Exception {
if (isElasticSearch || isH2Database) {
return; // Don't try to start elasticsearch server
}
File touch = new File(serverTouchDir);
if (touch.exists()) {
return;
}
File dir = new File(serverDir);
if (dir.isDirectory()) {
FileUtilities.deleteDir(dir);
}
String[] params = new String[2];
params[0] = "start";
params[1] = serverDir;
dir.mkdir(); // Create the data directory
new BSimControlLaunchable().run(params);
byte[] touchBytes = new byte[2];
touchBytes[0] = 'a';
touchBytes[1] = 'b';
FileUtilities.writeBytes(touch, touchBytes);
}
public void shutdownServer() throws Exception {
if (isElasticSearch || isH2Database) {
return;
}
File touch = new File(serverTouchDir);
if (!touch.exists()) {
return;
}
String[] params = new String[2];
params[0] = "stop";
params[1] = serverDir;
new BSimControlLaunchable().run(params);
touch.delete(); // Remove the touch file
File dir = new File(serverDir);
if (dir.isDirectory()) {
FileUtilities.deleteDir(dir); // Clean up database files
}
}
}
GhidraBuild/BuildFiles/Doclets/Module.manifest
-1
View File
@@ -1 +0,0 @@
MODULE FILE LICENSE: lib/json-simple-1.1.1.jar Apache License 2.0