Merge remote-tracking branch 'origin/GP-4456_ghidra1_ProjectFolderPerformance--SQUASHED'

This commit is contained in:
ghidra1
2024-03-25 20:07:06 -04:00
10 changed files with 187 additions and 130 deletions
@@ -74,6 +74,14 @@ public interface FileSystem {
*/
public String[] getItemNames(String folderPath) throws IOException;
/**
* Returns a list of the folder items contained in the given folder.
* @param folderPath the path of the folder.
* @return a list of folder items.
* @throws IOException
*/
public FolderItem[] getItems(String folderPath) throws IOException;
/**
* Returns the FolderItem in the given folder with the given name
* @param folderPath the folder path containing the item.
@@ -108,8 +116,8 @@ public interface FileSystem {
* all alphanumerics
* @throws IOException thrown if an IO error occurs.
*/
public void createFolder(String parentPath, String folderName) throws InvalidNameException,
IOException;
public void createFolder(String parentPath, String folderName)
throws InvalidNameException, IOException;
/**
* Create a new database item within the specified parent folder using the contents
@@ -133,8 +141,8 @@ public interface FileSystem {
*/
public DatabaseItem createDatabase(String parentPath, String name, String fileID,
BufferFile bufferFile, String comment, String contentType, boolean resetDatabaseId,
TaskMonitor monitor, String user) throws InvalidNameException, IOException,
CancelledException;
TaskMonitor monitor, String user)
throws InvalidNameException, IOException, CancelledException;
/**
* Create a new empty database item within the specified parent folder.
@@ -176,8 +184,8 @@ public interface FileSystem {
* @throws CancelledException if cancelled by monitor
*/
public DataFileItem createDataFile(String parentPath, String name, InputStream istream,
String comment, String contentType, TaskMonitor monitor) throws InvalidNameException,
IOException, CancelledException;
String comment, String contentType, TaskMonitor monitor)
throws InvalidNameException, IOException, CancelledException;
/**
* Creates a new file item from a packed file.
@@ -195,8 +203,8 @@ public interface FileSystem {
* @throws CancelledException if cancelled by monitor
*/
public FolderItem createFile(String parentPath, String name, File packedFile,
TaskMonitor monitor, String user) throws InvalidNameException, IOException,
CancelledException;
TaskMonitor monitor, String user)
throws InvalidNameException, IOException, CancelledException;
/**
* Delete the specified folder.
@@ -60,9 +60,8 @@ public interface FolderItem {
/**
* Return the file ID if one has been established or null
* @throws IOException thrown if IO or access error occurs
*/
String getFileID() throws IOException;
String getFileID();
/**
* Assign a new file-ID to this local non-versioned file.
@@ -288,8 +287,8 @@ public interface FolderItem {
* @throws IOException
* @throws CancelledException if monitor cancels operation
*/
public void output(File outputFile, int version, TaskMonitor monitor) throws IOException,
CancelledException;
public void output(File outputFile, int version, TaskMonitor monitor)
throws IOException, CancelledException;
/**
* Returns this instance after refresh or null if item no longer exists
@@ -604,9 +604,8 @@ public class IndexedLocalFileSystem extends LocalFileSystem {
deallocateItemStorage(parentPath, name);
}
finally {
Msg.warn(this,
"Detected orphaned project file " + conflictedItemStorageName + ": " +
getPath(parentPath, name));
Msg.warn(this, "Detected orphaned project file " + conflictedItemStorageName +
": " + getPath(parentPath, name));
}
}
@@ -893,8 +892,7 @@ public class IndexedLocalFileSystem extends LocalFileSystem {
}
@Override
protected String[] getItemNames(String folderPath, boolean includeHiddenFiles)
throws IOException {
public String[] getItemNames(String folderPath, boolean includeHiddenFiles) throws IOException {
if (readOnly) {
refreshReadOnlyIndex();
}
@@ -134,7 +134,8 @@ public class IndexedV1LocalFileSystem extends IndexedLocalFileSystem {
}
@Override
public FolderItem getItem(String fileID) throws IOException, UnsupportedOperationException {
public LocalFolderItem getItem(String fileID)
throws IOException, UnsupportedOperationException {
checkDisposed();
if (fileIdMap == null) {
return null;
@@ -366,13 +366,9 @@ public abstract class LocalFileSystem implements FileSystem {
protected abstract void deallocateItemStorage(String folderPath, String itemName)
throws IOException;
protected abstract String[] getItemNames(String folderPath, boolean includeHiddenFiles)
public abstract String[] getItemNames(String folderPath, boolean includeHiddenFiles)
throws IOException;
/**
*
* @see ghidra.framework.store.FileSystem#getItemNames(java.lang.String)
*/
@Override
public synchronized String[] getItemNames(String folderPath) throws IOException {
return getItemNames(folderPath, false);
@@ -407,10 +403,21 @@ public abstract class LocalFileSystem implements FileSystem {
}
@Override
public FolderItem getItem(String fileID) throws IOException, UnsupportedOperationException {
public LocalFolderItem getItem(String fileID)
throws IOException, UnsupportedOperationException {
throw new UnsupportedOperationException("getItem by File-ID");
}
@Override
public LocalFolderItem[] getItems(String folderPath) throws IOException {
String[] itemNames = getItemNames(folderPath, false);
LocalFolderItem[] folderItems = new LocalFolderItem[itemNames.length];
for (int i = 0; i < itemNames.length; i++) {
folderItems[i] = getItem(folderPath, itemNames[i]);
}
return folderItems;
}
@Override
public synchronized LocalDatabaseItem createDatabase(String parentPath, String name,
String fileID, BufferFile bufferFile, String comment, String contentType,
@@ -729,7 +736,7 @@ public abstract class LocalFileSystem implements FileSystem {
if (folderPath.length() == 1) {
return;
}
String[] items = getItemNames(folderPath);
String[] items = getItemNames(folderPath, false);
if (items.length > 0) {
return;
}
@@ -140,8 +140,7 @@ public class MangledLocalFileSystem extends LocalFileSystem {
// }
@Override
protected String[] getItemNames(String folderPath, boolean includeHiddenFiles)
throws IOException {
public String[] getItemNames(String folderPath, boolean includeHiddenFiles) throws IOException {
File dir = getFile(folderPath);
File[] dirList = dir.listFiles();
@@ -294,8 +293,7 @@ public class MangledLocalFileSystem extends LocalFileSystem {
*/
@Override
public synchronized void renameFolder(String parentPath, String folderName,
String newFolderName)
throws InvalidNameException, IOException {
String newFolderName) throws InvalidNameException, IOException {
if (readOnly) {
throw new ReadOnlyException();
@@ -440,16 +438,14 @@ public class MangledLocalFileSystem extends LocalFileSystem {
cleanupAfterConstruction(); // remove all temporary content
File tmpRoot =
new File(root.getCanonicalFile().getParentFile(), HIDDEN_DIR_PREFIX + '.' +
root.getName());
File tmpRoot = new File(root.getCanonicalFile().getParentFile(),
HIDDEN_DIR_PREFIX + '.' + root.getName());
if (tmpRoot.exists() || !tmpRoot.mkdir()) {
throw new IOException("Failed to create data directory: " + tmpRoot);
}
IndexedV1LocalFileSystem indexedFs =
new IndexedV1LocalFileSystem(tmpRoot.getAbsolutePath(), isVersioned, false, false,
true);
IndexedV1LocalFileSystem indexedFs = new IndexedV1LocalFileSystem(tmpRoot.getAbsolutePath(),
isVersioned, false, false, true);
migrationInProgress = true;
migrateFolder(SEPARATOR, indexedFs);
@@ -474,7 +470,7 @@ public class MangledLocalFileSystem extends LocalFileSystem {
indexedFs.createFolder(folderPath, name);
migrateFolder(getPath(folderPath, name), indexedFs);
}
for (String name : getItemNames(folderPath)) {
for (String name : getItemNames(folderPath, false)) {
LocalFolderItem item = getItem(folderPath, name);
indexedFs.migrateItem(item);
}
@@ -108,6 +108,19 @@ public class RemoteFileSystem implements FileSystem, RemoteAdapterListener {
return names;
}
@Override
public FolderItem[] getItems(String folderPath) throws IOException {
RepositoryItem[] items = repository.getItemList(folderPath);
FolderItem[] folderItems = new FolderItem[items.length];
for (int i = 0; i < items.length; i++) {
if (items[i].getItemType() != RepositoryItem.DATABASE) {
throw new IOException("Unsupported file type");
}
folderItems[i] = new RemoteDatabaseItem(repository, items[i]);
}
return folderItems;
}
@Override
public synchronized FolderItem getItem(String folderPath, String name) throws IOException {
RepositoryItem item = repository.getItem(folderPath, name);
@@ -907,15 +907,17 @@ public class DefaultProjectData implements ProjectData {
private void findCheckedOutFiles(String folderPath, List<DomainFile> checkoutList,
TaskMonitor monitor) throws IOException, CancelledException {
for (String name : fileSystem.getItemNames(folderPath)) {
DomainFolder folder = getFolder(folderPath);
if (folder == null) {
return;
}
GhidraFolderData folderData = getRootFolderData().getFolderPathData(folderPath, false);
for (String name : folderData.getFileNames()) {
monitor.checkCancelled();
LocalFolderItem item = fileSystem.getItem(folderPath, name);
if (item.getCheckoutId() != FolderItem.DEFAULT_CHECKOUT_ID) {
GhidraFolderData folderData =
getRootFolderData().getFolderPathData(folderPath, false);
if (folderData != null) {
checkoutList.add(new GhidraFile(folderData.getDomainFolder(), name));
}
GhidraFileData fileData = folderData.getFileData(name, false);
if (fileData != null && fileData.isCheckedOut()) {
checkoutList.add(new GhidraFile(folderData.getDomainFolder(), name));
}
}
@@ -113,6 +113,46 @@ public class GhidraFileData {
refresh();
}
/**
* Construct a new file instance with a specified name and a corresponding parent folder using
* up-to-date folder items.
* @param parent parent folder
* @param name file name
* @param folderItem local folder item
* @param versionedFolderItem versioned folder item
*/
GhidraFileData(GhidraFolderData parent, String name, LocalFolderItem folderItem,
FolderItem versionedFolderItem) {
this.parent = parent;
this.name = name;
this.folderItem = folderItem;
this.versionedFolderItem = versionedFolderItem;
this.projectData = parent.getProjectData();
this.fileSystem = parent.getLocalFileSystem();
this.versionedFileSystem = parent.getVersionedFileSystem();
this.listener = parent.getChangeListener();
validateCheckout();
updateFileID();
}
void refresh(LocalFolderItem localFolderItem, FolderItem verFolderItem) {
icon = null;
disabledIcon = null;
this.folderItem = localFolderItem;
this.versionedFolderItem = verFolderItem;
validateCheckout();
boolean fileIDset = updateFileID();
if (parent.visited()) {
// NOTE: we should maintain some cached data so we can determine if something really changed
listener.domainFileStatusChanged(getDomainFile(), fileIDset);
}
}
private boolean refresh() throws IOException {
String parentPath = parent.getPathname();
if (folderItem == null) {
@@ -138,9 +178,12 @@ public class GhidraFileData {
if (folderItem == null && versionedFolderItem == null) {
throw new FileNotFoundException(name + " not found");
}
return updateFileID();
}
private boolean updateFileID() {
boolean fileIdWasNull = fileID == null;
fileID = folderItem != null ? folderItem.getFileID() : versionedFolderItem.getFileID();
return fileIdWasNull && fileID != null;
}
@@ -157,26 +200,32 @@ public class GhidraFileData {
disabledIcon = null;
fileIDset |= refresh();
if (parent.visited()) {
// NOTE: we should maintain some cached data so we can determine if something really changed
listener.domainFileStatusChanged(getDomainFile(), fileIDset);
}
}
private void validateCheckout() throws IOException {
private void validateCheckout() {
if (fileSystem.isReadOnly() || !versionedFileSystem.isOnline()) {
return;
}
if (folderItem != null && folderItem.isCheckedOut()) {
// Cleanup checkout status which may be stale
if (versionedFolderItem != null) {
ItemCheckoutStatus coStatus =
versionedFolderItem.getCheckout(folderItem.getCheckoutId());
if (coStatus == null) {
try {
if (folderItem != null && folderItem.isCheckedOut()) {
// Cleanup checkout status which may be stale
if (versionedFolderItem != null) {
ItemCheckoutStatus coStatus =
versionedFolderItem.getCheckout(folderItem.getCheckoutId());
if (coStatus == null) {
folderItem.clearCheckout();
}
}
else {
folderItem.clearCheckout();
}
}
else {
folderItem.clearCheckout();
}
}
catch (IOException e) {
// ignore
}
}
@@ -24,8 +24,10 @@ import ghidra.framework.model.*;
import ghidra.framework.protocol.ghidra.GhidraURL;
import ghidra.framework.protocol.ghidra.TransientProjectData;
import ghidra.framework.store.FileSystem;
import ghidra.framework.store.FolderItem;
import ghidra.framework.store.FolderNotEmptyException;
import ghidra.framework.store.local.LocalFileSystem;
import ghidra.framework.store.local.LocalFolderItem;
import ghidra.util.*;
import ghidra.util.exception.*;
import ghidra.util.task.TaskMonitor;
@@ -60,7 +62,7 @@ class GhidraFolderData {
// folderList and fileList are only be used if visited is true
private Set<String> folderList = new TreeSet<>();
private Set<String> fileList = new TreeSet<>();
private boolean visited; // true if full refresh was performed
private Map<String, GhidraFileData> fileDataCache = new HashMap<>();
@@ -172,7 +174,7 @@ class GhidraFolderData {
/**
* Get folder data for specified absolute or relative folderPath
* @param folderPath
* @param folderPath absolute or relative folder path
* @param lazy if true folder will not be searched for if not already discovered - in
* this case null will be returned
* @return folder data or null if not found or lazy=true and not yet discovered
@@ -236,9 +238,10 @@ class GhidraFolderData {
}
updateExistenceState();
checkInUse();
boolean sendEvent = true;
String oldName = name;
String parentPath = parent.getPathname();
if (folderExists) {
fileSystem.renameFolder(parentPath, name, newName);
}
@@ -247,8 +250,8 @@ class GhidraFolderData {
versionedFileSystem.renameFolder(parentPath, name, newName);
}
catch (IOException e) {
sendEvent = false;
if (folderExists) {
// revert local folder name
fileSystem.renameFolder(parentPath, newName, name);
}
throw e;
@@ -260,18 +263,14 @@ class GhidraFolderData {
name = newName;
parent.folderDataCache.put(newName, this);
fileDataCache.clear();
folderDataCache.clear();
GhidraFolder newFolder = getDomainFolder();
if (parent.visited) {
parent.folderList.remove(oldName);
parent.folderList.add(newName);
if (sendEvent) {
listener.domainFolderRenamed(newFolder, oldName);
}
listener.domainFolderRenamed(newFolder, oldName);
}
return newFolder;
}
}
@@ -322,7 +321,7 @@ class GhidraFolderData {
boolean isEmpty() {
try {
refresh(false, false, null); // visited will be true upon return
return folderList.isEmpty() && fileList.isEmpty();
return folderList.isEmpty() && fileDataCache.isEmpty();
}
catch (IOException e) {
// TODO: what should we return if folder not found or error occurs?
@@ -343,7 +342,7 @@ class GhidraFolderData {
Msg.error(this, "Folder refresh failed: " + e.getMessage());
return new ArrayList<>();
}
return new ArrayList<>(fileList);
return new ArrayList<>(fileDataCache.keySet());
}
/**
@@ -375,12 +374,6 @@ class GhidraFolderData {
!newFileName.equals(fileData.getName())) {
throw new AssertException();
}
if (visited) {
fileList.remove(oldFileName);
}
if (visited) {
fileList.add(newFileName);
}
fileDataCache.put(newFileName, fileData);
if (visited) {
listener.domainFileRenamed(getDomainFile(newFileName), oldFileName);
@@ -404,12 +397,6 @@ class GhidraFolderData {
!newFileName.equals(fileData.getName())) {
throw new AssertException();
}
if (visited) {
fileList.remove(oldFileName);
}
if (newParent.visited) {
newParent.fileList.add(newFileName);
}
newParent.fileDataCache.put(newFileName, fileData);
}
if (visited || newParent.visited) {
@@ -436,7 +423,6 @@ class GhidraFolderData {
fileData.dispose();
fileDataCache.remove(fileName);
if (visited) {
fileList.remove(fileName);
listener.domainFileRemoved(getDomainFolder(), fileName, fileID);
}
}
@@ -445,21 +431,13 @@ class GhidraFolderData {
if (visited) {
try {
fileData = addFileData(fileName);
if (fileData != null) {
listener.domainFileAdded(fileData.getDomainFile());
}
}
catch (IOException e) {
// ignore
}
if (fileData == null) {
if (fileList.remove(fileName)) {
listener.domainFileRemoved(getDomainFolder(), fileName, null);
}
}
else if (fileList.add(fileName)) {
listener.domainFileAdded(fileData.getDomainFile());
}
else {
listener.domainFileStatusChanged(fileData.getDomainFile(), false);
}
}
}
}
@@ -532,7 +510,6 @@ class GhidraFolderData {
void dispose() {
visited = false;
folderList.clear();
fileList.clear();
for (GhidraFolderData folderData : folderDataCache.values()) {
folderData.dispose();
}
@@ -636,17 +613,28 @@ class GhidraFolderData {
}
}
private <T extends FolderItem> Map<String, T> itemMapOf(T[] items) {
Map<String, T> map = new HashMap<>();
for (T item : items) {
if (item != null) {
map.put(item.getName(), item);
}
}
return map;
}
private void refreshFiles(TaskMonitor monitor) throws IOException {
String path = getPathname();
boolean hadError = false;
Map<String, LocalFolderItem> localItemMap = Map.of();
Map<String, FolderItem> versionedItemMap = Map.of();
HashSet<String> newSet = new HashSet<>();
if (folderExists) {
try {
String[] items = fileSystem.getItemNames(path);
newSet.addAll(Arrays.asList(items));
localItemMap = itemMapOf(fileSystem.getItems(path));
newSet.addAll(localItemMap.keySet());
}
catch (IOException e) {
if (parent != null) {
@@ -657,8 +645,8 @@ class GhidraFolderData {
}
if (versionedFolderExists) {
try {
String[] items = versionedFileSystem.getItemNames(path);
newSet.addAll(Arrays.asList(items));
versionedItemMap = itemMapOf(versionedFileSystem.getItems(path));
newSet.addAll(versionedItemMap.keySet());
}
catch (Exception e) {
Msg.error(this, "versioned folder refresh failed: " + e.getMessage());
@@ -667,7 +655,7 @@ class GhidraFolderData {
}
HashSet<String> oldSet = new HashSet<>();
for (String file : fileList) {
for (String file : fileDataCache.keySet()) {
oldSet.add(file);
}
HashSet<String> oldSetClone = new HashSet<>(oldSet);
@@ -679,23 +667,15 @@ class GhidraFolderData {
}
// refresh existing
for (String fileName : fileList.toArray(new String[fileList.size()])) {
GhidraFileData fileData = fileDataCache.get(fileName);
if (fileData != null) {
try {
fileData.statusChanged();
}
catch (IOException e) {
if (!(e instanceof FileNotFoundException)) {
if (hadError) {
throw e;
}
hadError = true; // tolerate single file error and remove file reference
Msg.error(this,
"Domain File error on " + fileData.getPathname() + ": " + e.toString());
}
fileRemoved(fileName);
}
for (GhidraFileData fileData : fileDataCache.values()) {
String fileName = fileData.getName();
LocalFolderItem localFolderItem = localItemMap.get(fileName);
FolderItem versionedFolderItem = versionedItemMap.get(fileName);
if (localFolderItem == null && versionedFolderItem == null) {
fileRemoved(fileName);
}
else {
fileData.refresh(localItemMap.get(fileName), versionedItemMap.get(fileName));
}
}
@@ -705,12 +685,12 @@ class GhidraFolderData {
if (monitor != null && monitor.isCancelled()) {
break;
}
GhidraFileData fileData = addFileData(fileName);
if (fileData != null) {
fileList.add(fileName);
if (visited) {
listener.domainFileAdded(fileData.getDomainFile());
}
LocalFolderItem localFolderItem = localItemMap.get(fileName);
FolderItem versionedFolderItem = versionedItemMap.get(fileName);
GhidraFileData fileData = addFileData(fileName, localFolderItem, versionedFolderItem);
if (visited) {
listener.domainFileAdded(fileData.getDomainFile());
}
}
}
@@ -722,7 +702,6 @@ class GhidraFolderData {
fileID = fileData.getFileID();
fileData.dispose();
}
fileList.remove(filename);
if (visited) {
listener.domainFileRemoved(getDomainFolder(), filename, fileID);
}
@@ -860,7 +839,7 @@ class GhidraFolderData {
return true;
}
if (visited) {
return fileList.contains(fileName);
return false;
}
return addFileData(fileName) != null;
}
@@ -888,6 +867,15 @@ class GhidraFolderData {
return fileData;
}
private GhidraFileData addFileData(String fileName, LocalFolderItem folderItem,
FolderItem versionedFolderItem) {
GhidraFileData fileData =
new GhidraFileData(this, fileName, folderItem, versionedFolderItem);
fileDataCache.put(fileName, fileData);
projectData.updateFileIndex(fileData);
return fileData;
}
/**
* Get file data for child specified by fileName
* @param fileName name of file
@@ -1101,7 +1089,7 @@ class GhidraFolderData {
if (fileSystem.getFolderNames(path).length != 0) {
return;
}
if (fileSystem.getItemNames(path).length != 0) {
if (fileSystem.getItemNames(path, false).length != 0) {
return;
}
delete();
@@ -1134,7 +1122,6 @@ class GhidraFolderData {
throw new IllegalArgumentException("newParent must differ from current parent");
}
checkInUse();
boolean sendEvent = true;
updateExistenceState();
try {
@@ -1152,8 +1139,8 @@ class GhidraFolderData {
newParent.getPathname());
}
catch (IOException e) {
sendEvent = false;
if (folderExists) {
// revert local folder move
fileSystem.moveFolder(newParent.getPathname(), name,
parent.getPathname());
}
@@ -1168,9 +1155,6 @@ class GhidraFolderData {
}
parent.folderDataCache.remove(name);
fileDataCache.clear();
folderDataCache.clear();
if (newParent.visited) {
newParent.folderList.add(name);
}
@@ -1179,7 +1163,7 @@ class GhidraFolderData {
parent = newParent;
GhidraFolder newFolder = getDomainFolder();
if (sendEvent && (parent.visited || newParent.visited)) {
if (parent.visited || newParent.visited) {
listener.domainFolderMoved(newFolder, oldParent);
}