diff --git a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java index 8f487d6f05..e69bd9883e 100644 --- a/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java +++ b/Ghidra/Debug/Debugger/src/main/java/ghidra/app/plugin/core/debug/disassemble/DebuggerDisassemblerPlugin.java @@ -22,10 +22,10 @@ import docking.ActionContext; import docking.Tool; import docking.action.DockingActionIf; import docking.actions.PopupActionProvider; -import generic.jar.ResourceFile; import ghidra.app.plugin.PluginCategoryNames; import ghidra.app.plugin.core.debug.DebuggerPluginPackage; import ghidra.app.plugin.core.debug.gui.listing.DebuggerListingActionContext; +import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription; import ghidra.app.services.DebuggerPlatformService; import ghidra.app.services.DebuggerTraceManagerService; import ghidra.framework.plugintool.*; @@ -202,27 +202,17 @@ public class DebuggerDisassemblerPlugin extends Plugin implements PopupActionPro protected Collection getAlternativeLanguageIDs(Language language) { // One of the alternatives is the language's actual default LanguageDescription desc = language.getLanguageDescription(); - if (!(desc instanceof SleighLanguageDescription)) { + if (!(desc instanceof SleighLanguageDescription sld)) { return List.of(); } - SleighLanguageDescription sld = (SleighLanguageDescription) desc; - ResourceFile slaFile = sld.getSlaFile(); List result = new ArrayList<>(); LanguageService langServ = DefaultLanguageService.getLanguageService(); for (LanguageDescription altDesc : langServ.getLanguageDescriptions(false)) { - if (!(altDesc instanceof SleighLanguageDescription)) { - continue; + if (altDesc instanceof SleighLanguageDescription altSld && + sld.isSameSleighLanguageFile(altSld) && sld.getEndian() == altSld.getEndian()) { + result.add(altSld.getLanguageID()); } - SleighLanguageDescription altSld = (SleighLanguageDescription) altDesc; - if (!altSld.getSlaFile().equals(slaFile)) { - continue; - } - if (altSld.getEndian() != sld.getEndian()) { - // Memory endian, not necessarily instruction endian - continue; - } - result.add(altSld.getLanguageID()); } return result; } diff --git a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java index 468a8a0803..02e795ecfa 100644 --- a/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java +++ b/Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/guest/DBTraceGuestPlatform.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.tuple.Pair; import db.DBRecord; import generic.jar.ResourceFile; +import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription; import ghidra.app.util.PseudoInstruction; import ghidra.framework.data.OpenMode; import ghidra.lifecycle.Internal; @@ -304,7 +305,7 @@ public class DBTraceGuestPlatform extends DBAnnotatedObject private static ResourceFile getSlaFile(Language language) { SleighLanguageDescription desc = (SleighLanguageDescription) language.getLanguageDescription(); - return desc.getSlaFile(); + return desc.getLanguageFile().getSlaFile(); } @Override diff --git a/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java b/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java index cbb51d82e4..b5c5aae5a7 100644 --- a/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java +++ b/Ghidra/Extensions/SleighDevTools/src/main/java/ghidra/app/util/disassemble/GNUExternalDisassembler.java @@ -23,6 +23,7 @@ import org.jdom2.*; import org.jdom2.input.SAXBuilder; import generic.jar.ResourceFile; +import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription; import ghidra.app.util.bin.ByteProvider; import ghidra.app.util.bin.MemoryByteProvider; import ghidra.framework.*; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java index 005178f7a1..23f140ce0f 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/help/AboutProgramPlugin.java @@ -27,6 +27,7 @@ import ghidra.app.CorePluginPackage; import ghidra.app.context.ProgramActionContext; import ghidra.app.context.ProgramContextAction; import ghidra.app.plugin.PluginCategoryNames; +import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription; import ghidra.app.util.GenericHelpTopics; import ghidra.app.util.HelpTopics; import ghidra.framework.main.ApplicationLevelPlugin; @@ -169,7 +170,7 @@ public class AboutProgramPlugin extends Plugin implements ApplicationLevelPlugin metadata.put("Language Spec", lDesc.getDefsFile() + (lav.isMismatch() ? lav.getVersionDisplay() : "")); metadata.put("Processor Spec", lDesc.getSpecFile().getAbsolutePath()); - metadata.put("Sleigh Spec", lDesc.getSlaFile().getAbsolutePath() + "spec"); + metadata.put("Sleigh Spec", lDesc.getLanguageFile().getSlaSpecFile().getAbsolutePath()); } if (lav.compilerSpec != null) { metadata.put("Compiler Spec", diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java index 78c4f6277e..f2436a9216 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFUtil.java @@ -27,6 +27,7 @@ import java.util.regex.Pattern; import generic.jar.ResourceFile; import ghidra.app.cmd.comments.AppendCommentCmd; +import ghidra.app.plugin.processors.sleigh.SleighLanguageDescription; import ghidra.app.util.bin.format.dwarf.attribs.DWARFAttributeValue; import ghidra.app.util.bin.format.dwarf.attribs.DWARFNumericAttribute; import ghidra.app.util.bin.format.dwarf.expression.DWARFExpressionException; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java index be51454486..8f3545cbf1 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/task/ProgramOpener.java @@ -170,8 +170,12 @@ public class ProgramOpener { // we don't care, the task has been cancelled } catch (LanguageNotFoundException e) { - Msg.showError(this, null, "Error Opening " + filename, - e.getMessage() + "\nPlease contact the Ghidra team for assistance."); + String msg = e.getMessage() + "\n"; + if (e.getCause() != null) { + msg += e.getCause().getMessage() + "\n"; + } + msg += "Please contact the Ghidra team for assistance."; + Msg.showError(this, null, "Error Opening " + filename, msg); } catch (Exception e) { if (domainFile.isInWritableProject() && (e instanceof IOException)) { diff --git a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java index 25622a2c93..45fa1b9657 100644 --- a/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java +++ b/Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompInterface.java @@ -25,8 +25,7 @@ import java.util.ArrayList; import generic.jar.ResourceFile; import ghidra.app.decompiler.signature.DebugSignature; import ghidra.app.decompiler.signature.SignatureResult; -import ghidra.app.plugin.processors.sleigh.SleighLanguage; -import ghidra.app.plugin.processors.sleigh.UniqueLayout; +import ghidra.app.plugin.processors.sleigh.*; import ghidra.program.model.address.*; import ghidra.program.model.lang.*; import ghidra.program.model.listing.Function; @@ -278,8 +277,7 @@ public class DecompInterface { xmlEncode.clear(); dtmanage.encodeCoreTypes(xmlEncode); String coretypes = xmlEncode.toString(); - SleighLanguageDescription sleighdescription = - (SleighLanguageDescription) pcodelanguage.getLanguageDescription(); + SleighLanguageDescription sleighdescription = pcodelanguage.getLanguageDescription(); ResourceFile pspecfile = sleighdescription.getSpecFile(); String pspecxml = fileToString(pspecfile); xmlEncode.clear(); diff --git a/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java b/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java index 89432c7316..22593d0b02 100644 --- a/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java +++ b/Ghidra/Framework/Emulation/src/test/java/ghidra/app/plugin/processors/sleigh/SleighLanguageHelper.java @@ -88,7 +88,8 @@ public class SleighLanguageHelper { ); langDesc.setDefsFile(lDefsFile); langDesc.setSpecFile(pSpecFile); - langDesc.setSlaFile(slaFile); + langDesc.setLanguageFile( + SleighLanguageFile.fromSlaFilename(slaFile.getParentFile(), slaFile.getName())); MOCK_BE_64_LANGUAGE = new SleighLanguage(langDesc); return MOCK_BE_64_LANGUAGE; diff --git a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java index 9a3170f9b6..0823876dea 100644 --- a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java +++ b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/IOResult.java @@ -25,29 +25,78 @@ import utilities.util.reflection.ReflectionUtilities; import utility.function.Dummy; /** - * Class to pass to a thread pool that will consume all output from an external process. This is - * a {@link Runnable} that get submitted to a thread pool. This class records the data it reads + * {@link Runnable} that will consume all text output from an {@link InputStream} tied to an + * external processes (stdout / stderr). + *

+ * The output can be inspected line-by-line by providing a string {@link Consumer}, or the entire + * output of the process can be inspected by calling {@link #getOutput()} or + * {@link #getOutputAsString()}. */ public class IOResult implements Runnable { public static final String THREAD_POOL_NAME = "I/O Thread Pool"; - private List outputLines = new ArrayList<>(); + private final List outputLines; private BufferedReader commandOutput; private final Throwable inception; - private Consumer consumer = Dummy.consumer(); + private final Consumer consumer; + /** + * Creates a {@link IOResult} that consumes the specified {@link InputStream}, saving it + * as text lines. + * + * @param input {@link InputStream} + */ public IOResult(InputStream input) { - this(ReflectionUtilities.createThrowableWithStackOlderThan(IOResult.class), input); + this(input, null, true, + ReflectionUtilities.createThrowableWithStackOlderThan(IOResult.class)); } - public IOResult(Throwable inception, InputStream input) { + /** + * Creates a {@link IOResult} that consumes the specified {@link InputStream}, saving it + * as text lines. + * + * @param input {@link InputStream} + * @param inception information about where this object was created + */ + public IOResult(InputStream input, Throwable inception) { + this(input, null, true, inception); + } + + /** + * Creates a {@link IOResult} that consumes the specified {@link InputStream}, handing each + * line to the {@link Consumer}. + *

+ * Example: {@code new IOResult(process.getInputStream(), s -> System.out.println(s), null);} + * + * @param input {@link InputStream} + * @param lineConsumer {@link Consumer string consumer} + * @param inception information about where this object was created + */ + public IOResult(InputStream input, Consumer lineConsumer, Throwable inception) { + this(input, lineConsumer, false, inception); + } + + /** + * Creates a {@link IOResult} that consumes the specified {@link InputStream}, handing each + * line to the {@link Consumer} and optionally storing each line for later retrieval. + * + * @param input {@link InputStream} + * @param lineConsumer {@link Consumer string consumer}, optional + * @param retainLines boolean flag, if true, the contents read from the InputStream will be + * available via {@link #getOutput()} and {@link #getOutputAsString()} + * @param inception information about where this object was created + */ + public IOResult(InputStream input, Consumer lineConsumer, boolean retainLines, + Throwable inception) { + this.outputLines = retainLines ? new ArrayList<>() : List.of(); + if (retainLines) { + lineConsumer = Dummy.ifNull(lineConsumer).andThen(outputLines::add); + } + this.consumer = lineConsumer; this.inception = inception; - commandOutput = new BufferedReader(new InputStreamReader(input)); - } - public void setConsumer(Consumer consumer) { - this.consumer = consumer; + commandOutput = new BufferedReader(new InputStreamReader(input)); } public String getOutputAsString() { @@ -68,8 +117,7 @@ public class IOResult implements Runnable { try { while ((line = commandOutput.readLine()) != null) { - consumer.accept(line); - outputLines.add(line); + consumer.accept(line); // this both adds to outputLines and calls the upstream consumer } } catch (Exception e) { diff --git a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java index 0563259bf6..86eddb98cd 100644 --- a/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java +++ b/Ghidra/Framework/Generic/src/main/java/generic/concurrent/io/ProcessConsumer.java @@ -53,8 +53,7 @@ public class ProcessConsumer { * @param lineConsumer the line consumer; may be null * @return the future that will be complete when all lines are read */ - public static Future consume(InputStream is, - Consumer lineConsumer) { + public static Future consume(InputStream is, Consumer lineConsumer) { lineConsumer = Dummy.ifNull(lineConsumer); @@ -62,9 +61,38 @@ public class ProcessConsumer { ReflectionUtilities.createThrowableWithStackOlderThan(ProcessConsumer.class)); GThreadPool pool = GThreadPool.getSharedThreadPool(IOResult.THREAD_POOL_NAME); - IOResult runnable = new IOResult(inception, is); - runnable.setConsumer(lineConsumer); + IOResult runnable = new IOResult(is, lineConsumer, true, inception); Future future = pool.submit(runnable, runnable); return future; } + + /** + * Reads the given input stream line-by-line, calling the given consumer. When the + * InputStream reaches EOF, a final {@code null} will be sent to the line consumer. + *

+ * The Inputstream is consumed via a Thread created just for this setup. + * + * @param is the input stream + * @param lineConsumer the line consumer; may be null + * @param processName descriptive name of process being monitored + */ + public static void monitorAndSignalEof(InputStream is, Consumer lineConsumer, + String processName) { + + Throwable inception = ReflectionUtilities.filterJavaThrowable( + ReflectionUtilities.createThrowableWithStackOlderThan(ProcessConsumer.class)); + + IOResult stdoutReader = new IOResult(is, lineConsumer, false, inception); + + Runnable threadRunnable = lineConsumer != null // change mode if null + ? () -> { + stdoutReader.run(); + lineConsumer.accept(null); // signal that eof was reached + } + : stdoutReader; + + Thread t = new Thread(threadRunnable, "IO Thread for " + processName); + t.setDaemon(true); + t.start(); + } } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageExtraMethods.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileException.java similarity index 57% rename from Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageExtraMethods.java rename to Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileException.java index c80872e4c2..130dffafe4 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageExtraMethods.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileException.java @@ -1,27 +1,31 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -//package ghidra.app.plugin.processors.sleigh; -// -//import ghidra.app.plugin.processors.sleigh.symbol.SymbolTable; -//import ghidra.program.model.address.AddressSpace; -//import ghidra.program.model.lang.Language; -// -//public interface SleighLanguage extends Language { -// public AddressSpace getDefaultSpace(); -// public SymbolTable getSymbolTable(); -// public DecisionNode getRootDecisionNode(); -//} +package ghidra.app.plugin.processors.sleigh; + +/** + * Thrown when error concerning a sleigh file is encountered + */ +public class SleighFileException extends SleighException { + + public SleighFileException(String message) { + super(message); + } + + public SleighFileException(String message, Throwable e) { + super(message, e); + } + +} diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileLockException.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileLockException.java new file mode 100644 index 0000000000..2fde6dfa95 --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighFileLockException.java @@ -0,0 +1,28 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.processors.sleigh; + +public class SleighFileLockException extends SleighFileException { + + public SleighFileLockException(String message) { + super(message); + } + + public SleighFileLockException(String message, Throwable e) { + super(message, e); + } + +} diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java index aa5a38ce71..5c05cd94bc 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java @@ -22,10 +22,10 @@ import java.math.BigInteger; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.antlr.runtime.RecognitionException; import org.xml.sax.*; import generic.jar.ResourceFile; @@ -36,7 +36,6 @@ import ghidra.app.plugin.processors.sleigh.expression.PatternValue; import ghidra.app.plugin.processors.sleigh.symbol.*; import ghidra.framework.Application; import ghidra.pcode.utils.SlaFormat; -import ghidra.pcodeCPort.slgh_compile.SleighCompileLauncher; import ghidra.program.model.address.*; import ghidra.program.model.lang.*; import ghidra.program.model.listing.DefaultProgramContext; @@ -44,10 +43,11 @@ import ghidra.program.model.mem.MemBuffer; import ghidra.program.model.mem.MemoryAccessException; import ghidra.program.model.pcode.*; import ghidra.program.model.util.ProcessorSymbolType; -import ghidra.sleigh.grammar.SleighPreprocessor; import ghidra.sleigh.grammar.SourceFileIndexer; import ghidra.util.*; import ghidra.util.exception.InvalidInputException; +import ghidra.util.exception.TimeoutException; +import ghidra.util.task.PreserveStateWrappingTaskMonitor; import ghidra.util.task.TaskMonitor; import ghidra.util.xml.SpecXmlUtils; import ghidra.xml.*; @@ -70,7 +70,7 @@ public class SleighLanguage implements Language { private int numSections = 0; // Number of named sections for this language private int alignment = 1; private int defaultPointerWordSize = 1; // Default wordsize to send down with pointer data-types - private SleighLanguageDescription description; + private final SleighLanguageDescription description; private ParallelInstructionLanguageHelper parallelHelper; private SourceFileIndexer indexer; //used to provide source file info for constructors @@ -99,11 +99,16 @@ public class SleighLanguage implements Language { private AddressSpace default_space; private List ctxsetting = new ArrayList<>(); private LinkedHashMap properties = new LinkedHashMap<>(); - SortedMap manual = null; + private SortedMap manual = null; - SleighLanguage(SleighLanguageDescription description) - throws DecoderException, SAXException, IOException { - initialize(description); + SleighLanguage(SleighLanguageDescription description) throws SleighException { + this(description, TaskMonitor.DUMMY); + } + + SleighLanguage(SleighLanguageDescription description, TaskMonitor monitor) + throws SleighException { + this.description = description; + initialize(false, monitor); } private void addAdditionInject(InjectPayloadSleigh payload) { @@ -113,37 +118,68 @@ public class SleighLanguage implements Language { additionalInject.add(payload); } - private void initialize(SleighLanguageDescription langDescription) - throws DecoderException, SAXException, IOException { + private void initialize(boolean forceCompile, TaskMonitor monitor) throws SleighException { + long startTS = System.currentTimeMillis(); + this.defaultSymbols = new ArrayList<>(); this.defaultMemoryBlocks = new MemoryBlockDefinition[0]; this.compilerSpecDescriptions = new LinkedHashMap<>(); - for (CompilerSpecDescription compilerSpecDescription : langDescription + for (CompilerSpecDescription compilerSpecDescription : description .getCompatibleCompilerSpecDescriptions()) { this.compilerSpecDescriptions.put(compilerSpecDescription.getCompilerSpecID(), (SleighCompilerSpecDescription) compilerSpecDescription); } compilerSpecs = new HashMap<>(); - this.description = langDescription; additionalInject = null; - SleighLanguageValidator.validatePspecFile(langDescription.getSpecFile()); + SleighLanguageValidator.validatePspecFile(description.getSpecFile()); readInitialDescription(); - // should addressFactory and registers initialization be done at - // construction time? + // should addressFactory and registers initialization be done at construction time? // for now we'll assume yes. contextcache = new ContextCache(); - - ResourceFile slaFile = langDescription.getSlaFile(); - if (!slaFile.exists() || - (slaFile.canWrite() && (isSLAWrongVersion(slaFile) || isSLAStale(slaFile)))) { - reloadLanguage(TaskMonitor.DUMMY, true); + + SleighLanguageFile langFile = description.getLanguageFile(); + if (!langFile.getSlaSpecFile().exists()) { + throw new SleighFileException("Missing slaspec: " + langFile.getSlaSpecFile()); } + // check .sla file freshness inside lock, and recompile if necessary before releasing lock. + // if can't lock, it's single jar mode and we can't recompile anyways + AtomicLong lockElapsed = new AtomicLong(); + if (langFile.canLock()) { + try (PreserveStateWrappingTaskMonitor tm = + new PreserveStateWrappingTaskMonitor(monitor)) { + tm.setCancelEnabled(true); + tm.setShowProgressValue(true); + langFile.withLock(SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT, tm, () -> { + tm.setCancelEnabled(false); + long lockStartTS = System.currentTimeMillis(); + if (forceCompile || langFile.needsCompilation(SlaFormat.FORMAT_VERSION)) { + langFile.compileSlaFile(monitor); + } + lockElapsed.set(System.currentTimeMillis() - lockStartTS); + }); + } + catch (TimeoutException e) { + throw new SleighFileLockException( + "Timeout waiting for Sleigh language file lock: %s" + .formatted(langFile.getSlaFile()), + e); + } + catch (IOException e) { + throw new SleighFileException( + "Error locking Sleigh language file %s".formatted(langFile.getSlaFile()), e); + } + } + // Read in the sleigh specification - PackedDecode decoder = SlaFormat.buildDecoder(slaFile); - decode(decoder); + try (PackedDecode decoder = SlaFormat.buildDecoder(langFile.getSlaFile())) { + decode(decoder); + } + catch (IOException | DecoderException e) { + throw new SleighException("Error decoding", e); + } registerBuilder = new RegisterBuilder(); loadRegisters(registerBuilder); @@ -154,6 +190,10 @@ public class SleighLanguage implements Language { instructProtoMap = new ConcurrentHashMap<>(); initParallelHelper(); + + long initElapsed = System.currentTimeMillis() - startTS; + Msg.debug(this, "Took %dms (%dms inside lock) to initialize language %s" + .formatted(initElapsed, lockElapsed.get(), langFile)); } private void buildVolatileSymbolAddresses() { @@ -165,37 +205,6 @@ public class SleighLanguage implements Language { } } - private boolean isSLAWrongVersion(ResourceFile slaFile) { - try (InputStream stream = slaFile.getInputStream()) { - return !SlaFormat.isSlaFormat(stream); - } - catch (Exception e) { - return true; - } - } - - private boolean isSLAStale(ResourceFile slaFile) { - String slafilename = slaFile.getName(); - int index = slafilename.lastIndexOf('.'); - String slabase = slafilename.substring(0, index); - String slaspecfilename = slabase + ".slaspec"; - ResourceFile slaspecFile = new ResourceFile(slaFile.getParentFile(), slaspecfilename); - - File resourceAsFile = slaspecFile.getFile(true); - SleighPreprocessor preprocessor = - new SleighPreprocessor(new ModuleDefinitionsAdapter(), resourceAsFile); - long sourceTimestamp = Long.MAX_VALUE; - try { - sourceTimestamp = preprocessor.scanForTimestamp(); - } - catch (Exception e) { - // squash the error because we will force recompilation and errors - // will propagate elsewhere - } - long compiledTimestamp = slaFile.lastModified(); - return (sourceTimestamp > compiledTimestamp); - } - /** * Returns the unique base offset from which additional temporary variables * may be created. @@ -417,79 +426,11 @@ public class SleighLanguage implements Language { @Override public void reloadLanguage(TaskMonitor monitor) throws IOException { - reloadLanguage(monitor, false); - } - - private void reloadLanguage(TaskMonitor monitor, boolean calledFromInitialize) - throws IOException { - if (monitor == null) { - monitor = TaskMonitor.DUMMY; - } - monitor.setMessage("Compiling Language File..."); - - ResourceFile slaFile = description.getSlaFile(); - String slaName = slaFile.getName(); - int index = slaName.lastIndexOf('.'); - String specName = slaName.substring(0, index); - String languageName = specName + ".slaspec"; - ResourceFile languageFile = new ResourceFile(slaFile.getParentFile(), languageName); - - // see gradle/processorUtils.gradle for sleighArgs.txt generation - ResourceFile sleighArgsFile = null; - ResourceFile languageModule = Application.getModuleContainingResourceFile(languageFile); - if (languageModule != null) { - if (SystemUtilities.isInReleaseMode()) { - sleighArgsFile = new ResourceFile(languageModule, "data/sleighArgs.txt"); - } - else { - sleighArgsFile = new ResourceFile(languageModule, "build/tmp/sleighArgs.txt"); - } - } - - String[] args; - if (sleighArgsFile != null && sleighArgsFile.isFile()) { - String baseDir = Application.getInstallationDirectory() - .getAbsolutePath() - .replace(File.separatorChar, '/'); - if (!baseDir.endsWith("/")) { - baseDir += "/"; - } - args = new String[] { "-DBaseDir=" + baseDir, "-i", sleighArgsFile.getAbsolutePath(), - languageFile.getAbsolutePath(), description.getSlaFile().getAbsolutePath() }; - } - else { - args = new String[] { languageFile.getAbsolutePath(), - description.getSlaFile().getAbsolutePath() }; - } - try { - StringBuilder buf = new StringBuilder(); - for (String str : args) { - buf.append(str); - buf.append(" "); - } - Msg.debug(this, "Sleigh compile: " + buf); - int returnCode = SleighCompileLauncher.runMain(args); - if (returnCode != 0) { - throw new SleighException("Errors compiling " + languageFile.getAbsolutePath() + - " -- please check log messages for details"); - } + initialize(true, TaskMonitor.dummyIfNull(monitor)); } - catch (RecognitionException e) { - throw new IOException("RecognitionException error recompiling: " + e.getMessage()); - } - - if (!calledFromInitialize) { - monitor.setMessage("Reloading Language..."); - try { - initialize(description); - } - catch (DecoderException e) { - throw new IOException(e.getMessage()); - } - catch (SAXException e) { - throw new IOException(e.getMessage()); - } + catch (SleighException e) { + throw new IOException("Failed to reload Sleigh language", e); } } @@ -517,26 +458,33 @@ public class SleighLanguage implements Language { } }; - private void readInitialDescription() throws SAXException, IOException { - ResourceFile specFile = description.getSpecFile(); - XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false); + private void readInitialDescription() throws SleighException { try { - XmlElement nextElement = parser.peek(); - while (nextElement != null && !nextElement.getName().equals("segmented_address")) { - parser.next(); // skip element - nextElement = parser.peek(); - } - if (nextElement != null) { - XmlElement element = parser.start(); // segmented_address element - segmentedspace = element.getAttribute("space"); - segmentType = element.getAttribute("type"); - if (segmentType == null) { - segmentType = ""; + ResourceFile specFile = description.getSpecFile(); + XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false); + try { + XmlElement nextElement = parser.peek(); + while (nextElement != null && !nextElement.getName().equals("segmented_address")) { + parser.next(); // skip element + nextElement = parser.peek(); + } + if (nextElement != null) { + XmlElement element = parser.start(); // segmented_address element + segmentedspace = element.getAttribute("space"); + segmentType = element.getAttribute("type"); + if (segmentType == null) { + segmentType = ""; + } } } + finally { + parser.dispose(); + } } - finally { - parser.dispose(); + catch (SAXException | IOException e) { + throw new SleighException( + "Error reading initial description - language probably did not compile properly", + e); } } @@ -864,19 +812,26 @@ public class SleighLanguage implements Language { parser.end(el); } - private void readRemainingSpecification() throws SAXException, IOException { - ResourceFile specFile = description.getSpecFile(); - XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false); + private void readRemainingSpecification() throws SleighException { try { - read(parser); + ResourceFile specFile = description.getSpecFile(); + XmlPullParser parser = XmlPullParserFactory.create(specFile, SPEC_ERR_HANDLER, false); + try { + read(parser); + } + catch (XmlParseException e) { + Msg.error(this, "Failed to parse Sleigh Specification (" + specFile.getName() + + "): " + e.getMessage()); + } + finally { + parser.dispose(); + } } - catch (XmlParseException e) { - Msg.error(this, "Failed to parse Sleigh Specification (" + specFile.getName() + "): " + - e.getMessage()); - } - finally { - parser.dispose(); + catch (SAXException | IOException e) { + throw new SleighException( + "Error reading remaining spec - language probably did not compile properly", e); } + } private void decode(Decoder decoder) throws DecoderException { @@ -1163,7 +1118,7 @@ public class SleighLanguage implements Language { } @Override - public LanguageDescription getLanguageDescription() { + public SleighLanguageDescription getLanguageDescription() { return description; } @@ -1503,8 +1458,7 @@ public class SleighLanguage implements Language { } encoder.closeElement(ElementId.ELEM_SPACES); - SleighLanguageDescription sleighDescription = - (SleighLanguageDescription) getLanguageDescription(); + SleighLanguageDescription sleighDescription = getLanguageDescription(); Set truncatedSpaceNames = sleighDescription.getTruncatedSpaceNames(); if (!truncatedSpaceNames.isEmpty()) { for (String spaceName : truncatedSpaceNames) { diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/SleighLanguageDescription.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageDescription.java similarity index 72% rename from Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/SleighLanguageDescription.java rename to Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageDescription.java index 75b0a0adb9..63d2973b3e 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/SleighLanguageDescription.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageDescription.java @@ -4,30 +4,31 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package ghidra.program.model.lang; - -import generic.jar.ResourceFile; +package ghidra.app.plugin.processors.sleigh; import java.util.*; +import generic.jar.ResourceFile; +import ghidra.program.model.lang.*; + /** * Class for holding Language identifiers */ public class SleighLanguageDescription extends BasicLanguageDescription { private ResourceFile defsFile; // defs file - private ResourceFile specFile; // specification file - private ResourceFile slaFile; // just cramming this in here until major cleanup + private ResourceFile specFile; // pspec specification file private ResourceFile manualIndexFile; // the manual index file + private SleighLanguageFile languageFile; // sla, slaspec file private Map truncatedSpaceMap; @@ -36,8 +37,8 @@ public class SleighLanguageDescription extends BasicLanguageDescription { * @param id the name of the language * @param description language description text * @param processor processor name/family - * @param endian data endianness - * @param instructionEndian instruction endianness + * @param endian data endianess + * @param instructionEndian instruction endianess * @param size processor size * @param variant processor variant name * @param version the major version of the language. @@ -54,19 +55,15 @@ public class SleighLanguageDescription extends BasicLanguageDescription { Map> externalNames) { super(id, processor, endian, instructionEndian, size, variant, description, version, minorVersion, deprecated, compilerSpecDescriptions, externalNames); - this.specFile = null; - this.slaFile = null; - this.manualIndexFile = null; this.truncatedSpaceMap = spaceTruncations; } /** * @return set of address space names which have been identified for truncation */ - @SuppressWarnings("unchecked") public Set getTruncatedSpaceNames() { if (truncatedSpaceMap == null) { - return Collections.EMPTY_SET; + return Set.of(); } return truncatedSpaceMap.keySet(); } @@ -107,33 +104,37 @@ public class SleighLanguageDescription extends BasicLanguageDescription { * Set the (optional) specification file associated with this language * * @param specFile - * the specFile to associate with this description. + * the specFile (.pspec) to associate with this description. */ public void setSpecFile(ResourceFile specFile) { this.specFile = specFile; } /** - * Get the specification file (if it exists) + * Get the specification (.pspec) file (if it exists) * - * @return specification file + * @return specification file (.pspec) */ public ResourceFile getSpecFile() { return specFile; } /** - * @param slaFile + * Sets the {@link SleighLanguageFile} which represents the .sla and .slaspec files. + * + * @param langFile {@link SleighLanguageFile} which represents the .sla and .slaspec files */ - public void setSlaFile(ResourceFile slaFile) { - this.slaFile = slaFile; + void setLanguageFile(SleighLanguageFile langFile) { + this.languageFile = langFile; } /** - * @return + * Returns the {@link SleighLanguageFile} which represents the .sla and .slaspec files. + * + * @return {@link SleighLanguageFile} which represents the .sla and .slaspec files */ - public ResourceFile getSlaFile() { - return slaFile; + public SleighLanguageFile getLanguageFile() { + return languageFile; } public ResourceFile getManualIndexFile() { @@ -143,4 +144,16 @@ public class SleighLanguageDescription extends BasicLanguageDescription { public void setManualIndexFile(ResourceFile manualIndexFile) { this.manualIndexFile = manualIndexFile; } + + /** + * Tests if two Sleigh languages are based on the same .sla file. + * + * @param other {@link SleighLanguageDescription} + * @return true if the other {@link SleighLanguageDescription} uses the same .sla file + */ + public boolean isSameSleighLanguageFile(SleighLanguageDescription other) { + return other != null && + languageFile.getSlaFile().equals(other.getLanguageFile().getSlaFile()); + } + } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageFile.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageFile.java new file mode 100644 index 0000000000..775fed826a --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageFile.java @@ -0,0 +1,533 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.processors.sleigh; + +import static utilities.util.FileUtilities.*; + +import java.io.*; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import org.antlr.runtime.RecognitionException; +import org.apache.commons.io.FilenameUtils; + +import generic.jar.ResourceFile; +import ghidra.framework.Application; +import ghidra.pcode.utils.SlaFormat; +import ghidra.pcodeCPort.slgh_compile.SleighCompile; +import ghidra.pcodeCPort.slgh_compile.SleighCompileOptions; +import ghidra.sleigh.grammar.SleighPreprocessor; +import ghidra.util.SystemUtilities; +import ghidra.util.exception.TimeoutException; +import ghidra.util.task.TaskMonitor; +import utilities.util.FileResolutionResult; +import utilities.util.FileUtilities; + +/** + * Represents a Sleigh .sla and .slaspec file, and a way to lock them to ensure exclusive access + * while checking / updating the files. + */ +public class SleighLanguageFile { + public static final String SLASPEC_EXT = ".slaspec"; + public static final String SLA_EXT = ".sla"; + + /** + * Finds a sleigh language file, using search rules specific to sleigh. + *

+ * If file is not found in the specific base directory, the entire app install will be searched + * for a matching file. + * + * @param dir base directory where files are typically located + * @param filename name of file, with or without the extension + * @param expectedExtension extension of the specific type of sleigh file, leading dot required. + * Typically ".sla", ".slaspec", ".pspec", etc. + * @return ResourceFile that exists, never null + * @throws SleighFileException if file is not found or has bad case matching + */ + public static ResourceFile getLanguageResourceFile(ResourceFile dir, String filename, + String expectedExtension) throws SleighFileException { + ResourceFile f = findFile(dir, filename, expectedExtension); + if (f == null) { + f = new ResourceFile(dir, filename); + throw new SleighFileException( + "Missing sleigh file(%s): %s".formatted(expectedExtension, f.getAbsolutePath())); + } + + FileResolutionResult result = existsAndIsCaseDependent(f); + if (!result.isOk()) { + throw new SleighFileException("Sleigh file %s is not properly case dependent: %s" + .formatted(f.getAbsolutePath(), result.getMessage())); + } + return f; + } + + /** + * Creates a {@link SleighLanguageFile} instance using the language directory and sla filename + * to bootstrap the information about the sla file, slaspec file and lock file. + *

+ * NOTE: if the sla/slaspec are not found in the specified language directory, the entire + * application will be searched for the slaspec. + * + * @param dir {@link ResourceFile} language directory that should contain the sla or slaspec + * file + * @param slaFilename name of the sla file (typically from the ldefs xml value), with optional + * .sla file extension + * @return {@link SleighLanguageFile}, never null + * @throws SleighFileException if sla and slaspec file can not be found + */ + public static SleighLanguageFile fromSlaFilename(ResourceFile dir, String slaFilename) + throws SleighFileException { + String baseName = slaFilename.endsWith(SLA_EXT) + ? FilenameUtils.removeExtension(slaFilename) + : slaFilename; + + ResourceFile slaSpecRFile; + ResourceFile slaRFile; + try { + // find the slaspec and construct the sla filename from the slaspec's location + slaSpecRFile = getLanguageResourceFile(dir, baseName + SLASPEC_EXT, SLASPEC_EXT); + slaRFile = new ResourceFile(slaSpecRFile.getParentFile(), + FilenameUtils.removeExtension(slaSpecRFile.getName()) + SLA_EXT); + } + catch (SleighFileException e) { + try { + // if slaspec is not found, fall back and search for .sla file. + // if sla found, assume slaspec should be co-located there. + // This results in a SleighLanguageFile instance that allows + // the language description to be created, but the SleighLanguage + // will fail to initialize because of the missing .slaspec + slaRFile = getLanguageResourceFile(dir, baseName + SLA_EXT, SLA_EXT); + slaSpecRFile = new ResourceFile(slaRFile.getParentFile(), + FilenameUtils.removeExtension(slaRFile.getName()) + SLASPEC_EXT); + } + catch (SleighFileException e2) { + throw e; // throw original exception, not this one + } + } + if (slaSpecRFile.getFile(false) == null) { + // single jar mode, no locking, no compiling of sla file possible + + if (!slaRFile.exists()) { + throw new SleighFileException( + "Missing sleigh sla file: " + slaRFile.getAbsolutePath()); + } + return new SleighLanguageFile(slaRFile, slaSpecRFile, null); + } + + File lockFile = + new ResourceFile(slaRFile.getParentFile(), slaRFile.getName() + ".lock").getFile(false); + + return new SleighLanguageFile(slaRFile, slaSpecRFile, lockFile); + } + + /** + * Creates a {@link SleighLanguageFile} instance using the language directory and Sleigh + * language file name to bootstrap the information about the slaspec file and lock file. + *

+ * The .slaspec file will be found in the location indicated by the slaFilename. + *

+ * The actual .sla file will be private to the user and located under the user's + * .ghidra config directory. (or if single-jar mode, .sla will remain in the jar) + * + * @param dir {@link ResourceFile} language directory containing the slaspec file + * @param slaFilename name of the sla file (typically from the ldefs xml value), with optional + * .sla file extension + * @return {@link SleighLanguageFile}, never null + * @throws SleighFileException if slaspec is not found + */ + public static SleighLanguageFile fromSlaFilename_UserDir(ResourceFile dir, String slaFilename) + throws SleighFileException { + + String baseName = slaFilename.endsWith(SLA_EXT) + ? FilenameUtils.removeExtension(slaFilename) + : slaFilename; + + ResourceFile slaSpecRFile = + getLanguageResourceFile(dir, baseName + SLASPEC_EXT, SLASPEC_EXT); + if (slaSpecRFile.getFile(false) == null) { + // single jar mode, no locking, no compiling of sla file possible + ResourceFile slaRFile = new ResourceFile(slaSpecRFile.getParentFile(), + FilenameUtils.removeExtension(slaSpecRFile.getName()) + SLA_EXT); + if (!slaRFile.exists()) { + throw new SleighFileException( + "Missing sleigh sla file: " + slaRFile.getAbsolutePath()); + } + return new SleighLanguageFile(slaRFile, slaSpecRFile, null); + } + + File sleighUserDir = + new File(Application.getApplicationLayout().getUserSettingsDir(), "sleigh"); + if (!FileUtilities.mkdirs(sleighUserDir)) { + throw new SleighFileException("Bad user settings /sleigh directory: " + sleighUserDir); + } + + File slaFile = new File(sleighUserDir, + FilenameUtils.removeExtension(slaSpecRFile.getName()) + SLA_EXT); + + File lockFile = new File(sleighUserDir, slaFile.getName() + ".lock"); + return new SleighLanguageFile(new ResourceFile(slaFile), slaSpecRFile, lockFile); + } + + private final ResourceFile slaFile; + private final ResourceFile slaSpecFile; + private final File lockFile; + + private SleighLanguageFile(ResourceFile slaFile, ResourceFile slaSpecFile, File lockFile) { + this.slaFile = slaFile; + this.slaSpecFile = slaSpecFile; + this.lockFile = lockFile; + } + + /** + * Returns the path of this language's .sla file. + * + * @return path of the .sla file + */ + public ResourceFile getSlaFile() { + return slaFile; + } + + /** + * Returns the path of the .slaspec file. + * + * @return .slaspec file + */ + public ResourceFile getSlaSpecFile() { + return slaSpecFile; + } + + @Override + public String toString() { + return "%s -> %s".formatted(slaSpecFile.getAbsolutePath(), slaFile.getAbsolutePath()); + } + + /** + * Returns true if this language's files can be locked, or false if they can't be locked + * (embedded in a .jar file). + * + * @return true if this language's files can be locked + */ + public boolean canLock() { + return lockFile != null; + } + + /** + * Executes a runnable while a lock file is being held. + * + * @param Exception type thrown by the runnable + * @param timeout maximum amount of time to wait to acquire the lock file. This timeout does + * not apply to the Runnable that is executed once the lock is acquired. + * @param monitor {@link TaskMonitor} that will be updated with lock file acquire attempt info + * @param r a runnable that can throw a SleighException + * @throws E if runnable throws E + * @throws IOException IO error acquiring the lock file + * @throws TimeoutException if lock times out + */ + void withLock(Duration timeout, TaskMonitor monitor, + CheckedRunnable r) throws E, IOException, TimeoutException { + + if (lockFile == null) { + throw new IOException("Unable to lock language, missing lock file"); + } + + long timeoutMS = timeout.toMillis(); + long startts = System.currentTimeMillis(); + long maxts = startts + timeoutMS; + long sleepMS = Math.min(timeoutMS, 100); // 100ms is largest sleep-per-retry interval + long lockerPid = -1; + + monitor.initialize(timeoutMS / 1000, "Locking Sleigh language file"); + monitor.setProgress(timeoutMS / 1000); // run the progress meter as count down from max to 0 + + try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { + while (!monitor.isCancelled()) { + // Lock an unused/non-existent portion of the file to avoid read/write errors + // by other processes on Windows jvms + try (FileLock lock = raf.getChannel().tryLock(LOCKFILE_LOCK_OFFSET, 1, false)) { + if (lock != null) { + writeLockerInfo(raf); + r.run(); + raf.setLength(0); + return; + } + } + + // failed to acquire lock, sleep and try again + long remaining = maxts - System.currentTimeMillis(); + if (remaining < 0) { + // failed to acquire lock within allowed time, give up + break; + } + + lockerPid = tryReadLockerInfo(raf); + monitor.setMessage("Waiting on pid [%s] for Sleigh language file lock" + .formatted(lockerPid != -1 ? Long.toString(lockerPid) : "unknown")); + monitor.setProgress(remaining / 1000); + + Thread.sleep(sleepMS); + } + throw new TimeoutException( + "Timeout when trying to lock Sleigh language file [%s], locker's pid: [%s]" + .formatted(lockFile, + lockerPid != -1 ? Long.toString(lockerPid) : "unknown")); + } + catch (IOException | InterruptedException e) { + throw new IOException("Error locking Sleigh language file [%s]".formatted(lockFile), e); + } + } + + /** + * Checks if the sla file needs to be compiled/re-compiled. + *

+ * Conditions: missing .sla, .sla is older than the .slaspec, or the sla format version value + * inside the existing .sla file does not match the current sla format version. + *

+ * NOTE: this should only be called when holding the lock with + * {@link #withLock(Duration, TaskMonitor, CheckedRunnable)} + * + * @param requiredSlaFormatVersion required sla format version + * @return true if the .sla file needs to be compiled/recompiled + */ + public boolean needsCompilation(int requiredSlaFormatVersion) { + return !slaFile.exists() || isSlaFileStale() || getSlaVersion() != requiredSlaFormatVersion; + } + + /** + * Returns true if the slaspec file (or any included sinc files) is newer than the current + * sla file, indicating that the sla file should be recompiled. + *

+ * Returns false if the slaspec or sla files are embedded in a jar (eg. single-jar mode). + *

+ * NOTE: call this when holding the lock using {@link #withLock(Duration, TaskMonitor, CheckedRunnable)} + * + * @return true if slaspec file is newer than the sla file + */ + public boolean isSlaFileStale() { + // NOTE: SleighPreprocessor doesn't use ResourceFiles, so any 'include' directives processed + // by it won't use ResourceFile.getFile() + File f = slaSpecFile.getFile(false); + if (f == null) { + // if the slaspec file is embedded in a jar, always assume the sla file is correct + return false; + } + long slaSpecLastMod; + try { + SleighPreprocessor preprocessor = + new SleighPreprocessor(new ModuleDefinitionsAdapter(), f); + slaSpecLastMod = preprocessor.scanForTimestamp(); + } + catch (Exception e) { + // slaSpecLastMod will be max_value which will force recompilation, error parsing + // will be handled elsewhere + slaSpecLastMod = Long.MAX_VALUE; + } + long slaLastMod = slaFile.lastModified(); // will be 0 if does not exist + return slaLastMod == 0 || slaSpecLastMod > slaLastMod; + } + + /** + * Returns the format version number embedded in the compiled sla file. + *

+ * NOTE: this should only be called when holding the lock with + * {@link #withLock(Duration, TaskMonitor, CheckedRunnable)} + * + * @return format version number embedded in the sla file, or -1 if error reading or + * file doesn't exist + */ + public int getSlaVersion() { + try (InputStream stream = slaFile.getInputStream()) { + return SlaFormat.getSlaFormat(stream); + } + catch (Exception e) { + return -1; + } + } + + /** + * Compiles the slaspec file and replaces the sla file with the newly compiled sleigh output. + *

+ * NOTE: this should only be called when holding the lock with + * {@link #withLock(Duration, TaskMonitor, CheckedRunnable)} + * + * @param monitor {@link TaskMonitor} + * @throws SleighException if error occurs during compilation + */ + public void compileSlaFile(TaskMonitor monitor) throws SleighException { + monitor.setMessage("Compiling Language File..."); + monitor.setIndeterminate(true); + + // see gradle/processorUtils.gradle for sleighArgs.txt generation + String baseDir = FilenameUtils + .separatorsToUnix(Application.getInstallationDirectory().getAbsolutePath()); + if (!baseDir.endsWith("/")) { + baseDir += "/"; + } + + ResourceFile sleighArgsFile = getSleighArgsForSlaSpec(slaSpecFile); + SleighCompileOptions compileOptions = sleighArgsFile != null && sleighArgsFile.exists() + ? SleighCompileOptions.fromFile(sleighArgsFile.getFile(true)) + : new SleighCompileOptions(); + + compileOptions.addPreprocessorMacroDefinition("BaseDir", baseDir); + + File inputFile = slaSpecFile.getFile(true); + + File destFile = slaFile.getFile(true); + long mypid = ProcessHandle.current().pid(); + File destTmpFile = + new File(destFile.getParentFile(), destFile.getName() + ".%d.tmp".formatted(mypid)); + + SleighCompile compiler = new SleighCompile(); + compiler.setOptions(compileOptions); + + try { + int returnCode = compiler.run_compilation(inputFile.getPath(), destTmpFile.getPath()); + if (returnCode != 0) { + destTmpFile.delete(); + throw new SleighException( + "Errors compiling %s -- please check log messages for details" + .formatted(slaSpecFile)); + } + if (!destTmpFile.renameTo(destFile)) { + // atomically renaming the tmp file to replace the existing dest file will + // succeed(linux)/fail(windows) depending on OS. + // If it failed, manually remove the old file and then rename. + // Because this is a non-atomic operation, its best to do this when holding a fs lock + if (destFile.exists()) { + checkedDelete(destFile); + } + checkedRename(destTmpFile, destFile); + } + } + catch (IOException | RecognitionException e) { + throw new SleighException("Error compiling %s".formatted(slaSpecFile), e); + } + } + + public interface CheckedRunnable { + void run() throws E; + } + + private static void checkedDelete(File f) throws IOException { + if (!f.delete()) { + throw new IOException("Unable to delete previous file %s".formatted(f)); + } + } + + private static void checkedRename(File srcFile, File destFile) throws IOException { + if (!srcFile.renameTo(destFile)) { + throw new IOException( + "Failed to rename temp file [%s] to [%s]".formatted(srcFile, destFile)); + } + } + + /** + * Offset in the lock file of where to place the lock so it doesn't interfere with + * reading the contents of the file from processes that don't have the lock (typically only + * an issue on windows). + */ + private static final int LOCKFILE_LOCK_OFFSET = 1_000_000; + + /** + * 

+	 * Raw:     .*(\/|\\)\.\.?(\/|\\)|\.(\/|\\)|\.\.(\/|\\)
+	 * Parts:   .*(\/|\\)\.\.?(\/|\\) - optional text followed by a forward or back slash, 
+	 *                                  followed by one or two literal dots, followed
+	 *                                  by a forward or back slash
+	 *      OR
+	 *          \.(\/|\\)             - a literal dot followed by a forward or back slash
+	 *      OR 
+	 *          \.\.(\/|\\)           - two literal dots followed by a forward or back slash
+	 *
+ */ + private static final Pattern RELATIVE_PATHS_PATTERN = + Pattern.compile(".*(\\/|\\\\)\\.\\.?(\\/|\\\\)|\\.(\\/|\\\\)|\\.\\.(\\/|\\\\)"); + + private static String discardRelativePath(String str) { + return RELATIVE_PATHS_PATTERN.matcher(str).replaceFirst(""); + } + + private static void writeLockerInfo(RandomAccessFile raf) throws IOException { + long mypid = ProcessHandle.current().pid(); + raf.setLength(0); + raf.write("%d\n".formatted(mypid).getBytes(StandardCharsets.UTF_8)); + } + + private static long tryReadLockerInfo(RandomAccessFile raf) { + try { + byte[] buffer = new byte[64]; + raf.seek(0); + int bytesRead = raf.read(buffer); + if (bytesRead > 0 && buffer[bytesRead - 1] == '\n') { + // low-tech verification of the data by checking for a trailing \n + String s = new String(buffer, 0, bytesRead - 1, StandardCharsets.UTF_8); + long lockersPid = Long.parseLong(s); + return lockersPid; + } + } + catch (IOException | NumberFormatException e) { + // fall thru + } + return -1; + } + + private static ResourceFile getSleighArgsForSlaSpec(ResourceFile slaSpecFile) { + ResourceFile languageModule = Application.getModuleContainingResourceFile(slaSpecFile); + if (languageModule == null) { + return null; + } + return new ResourceFile(languageModule, + SystemUtilities.isInReleaseMode() ? "data/sleighArgs.txt" : "build/tmp/sleighArgs.txt"); + } + + private static ResourceFile findFile(ResourceFile parentDir, String fileNameOrRelativePath, + String extension) { + ResourceFile file = new ResourceFile(parentDir, fileNameOrRelativePath).getCanonicalFile(); + if (file.exists()) { + return file; + } + + String fileName = FilenameUtils.getName(fileNameOrRelativePath); + List files = findFiles(fileName, extension); + if (files.size() == 1) { + return files.get(0); + } + + String relativePath = discardRelativePath(fileNameOrRelativePath); + for (ResourceFile resourceFile : files) { + if (file.getAbsolutePath().endsWith(relativePath)) { + return resourceFile; + } + } + return null; + } + + private static List findFiles(String fileName, String extension) { + List matches = new ArrayList(); + List files = Application.findFilesByExtensionInApplication(extension); + for (ResourceFile resourceFile : files) { + if (resourceFile.getName().equals(fileName)) { + matches.add(resourceFile); + } + } + return matches; + } + +} diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java index b14f20e98c..2c8d2287ac 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProvider.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,19 +17,18 @@ package ghidra.app.plugin.processors.sleigh; import static utilities.util.FileUtilities.*; -import java.io.FileNotFoundException; import java.io.IOException; +import java.time.Duration; import java.util.*; -import java.util.regex.Pattern; import org.xml.sax.*; import generic.jar.ResourceFile; import ghidra.framework.Application; import ghidra.program.model.lang.*; -import ghidra.program.model.pcode.DecoderException; import ghidra.util.Msg; import ghidra.util.SystemUtilities; +import ghidra.util.task.TaskMonitor; import ghidra.util.xml.SpecXmlUtils; import ghidra.xml.*; import utilities.util.FileResolutionResult; @@ -39,32 +38,11 @@ import utilities.util.FileResolutionResult; * specifications */ public class SleighLanguageProvider implements LanguageProvider { - /** - * 
-	 * Raw:     .*(\/|\\)\.\.?(\/|\\)|\.(\/|\\)|\.\.(\/|\\)
-	 * Parts:   .*(\/|\\)\.\.?(\/|\\) - optional text followed by a forward or back slash, 
-	 *                                  followed by one or two literal dots, followed
-	 *                                  by a forward or back slash
-	 *      OR
-	 *          \.(\/|\\)             - a literal dot followed by a forward or back slash
-	 *      OR 
-	 *          \.\.(\/|\\)           - two literal dots followed by a forward or back slash
-	 *
+ * Returns a singleton instance of a {@link SleighLanguageProvider}. + * + * @return singleton {@link SleighLanguageProvider} */ - private static final Pattern RELATIVE_PATHS_PATTERN = - Pattern.compile(".*(\\/|\\\\)\\.\\.?(\\/|\\\\)|\\.(\\/|\\\\)|\\.\\.(\\/|\\\\)"); - - private final LinkedHashMap languages = - new LinkedHashMap(); - private final LinkedHashMap descriptions = - new LinkedHashMap(); - private int failureCount = 0; - - public final static String LANGUAGE_DIR_NAME = "languages"; - - private static SleighLanguageProvider instance; // sleigh language provider instance (singleton) - public static synchronized SleighLanguageProvider getSleighLanguageProvider() { if (instance == null) { instance = new SleighLanguageProvider(); @@ -72,6 +50,19 @@ public class SleighLanguageProvider implements LanguageProvider { return instance; } + private static SleighLanguageProvider instance; // sleigh language provider instance (singleton) + + /** + * Property that can be set as a jvm startup option to control the sla lock timeout duration. + *

+ * See {@link #LANGUAGE_LOCK_TIMEOUT}. + */ + public static final String LANGUAGE_LOCK_TIMEOUT_PROPNAME = + "ghidra.app.plugin.processors.sleigh.SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT_MS"; + + private final Map languages = new LinkedHashMap<>(); // preserve load order + private int failureCount = 0; + /** * Construct sleigh language provider (singleton use) */ @@ -81,7 +72,7 @@ public class SleighLanguageProvider implements LanguageProvider { } catch (Exception e) { Msg.error(SleighLanguageProvider.class, - "Sleigh language provider initiailization failed", e); + "Sleigh language provider initialization failed", e); } } @@ -109,8 +100,14 @@ public class SleighLanguageProvider implements LanguageProvider { } catch (SleighException e) { ++failureCount; - Msg.showError(this, null, "Problem loading " + file.getName(), - "Validation error: " + e.getMessage(), e); + if (e instanceof SleighFileException) { + // no need for stack trace + Msg.showError(this, null, "Problem loading " + file.getName(), e.getMessage()); + } + else { + Msg.showError(this, null, "Problem loading " + file.getName(), + "Validation error: " + e.getMessage(), e); + } } } @@ -121,83 +118,52 @@ public class SleighLanguageProvider implements LanguageProvider { @Override public boolean isLanguageLoaded(LanguageID languageId) { - return languages.get(languageId) != null; + return languages.getOrDefault(languageId, LanguageRec.NOT_FOUND).lang != null; } @Override - public Language getLanguage(LanguageID languageId) { - SleighLanguageDescription description = descriptions.get(languageId); - SleighLanguage lang = languages.get(languageId); - if (lang == null && description != null) { - try { - lang = new SleighLanguage(description); - languages.put(languageId, lang); - } - catch (SleighException e) { - Msg.showError(this, null, "Error", - "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e); - throw e; - } - catch (DecoderException e) { - Msg.showError(this, null, "Error", - "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e); - throw new SleighException("Format violation in the .sla file", e); - } - catch (FileNotFoundException e) { - Msg.showError(this, null, "Error", - "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e); - throw new SleighException( - "File not found - language probably did not compile properly", e); - } - catch (SAXException e) { - Msg.showError(this, null, "Error", - "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e); - throw new SleighException( - "SAXException - language probably did not compile properly", e); - } - catch (IOException e) { - Msg.showError(this, null, "Error", - "Can't read language spec " + description.getSlaFile().getAbsolutePath(), e); - throw new SleighException( - "IOException - language probably did not compile properly", e); + public SleighLanguage getLanguage(LanguageID languageId, TaskMonitor monitor) + throws LanguageNotFoundException { + LanguageRec langRec = languages.getOrDefault(languageId, LanguageRec.NOT_FOUND); + if (langRec != LanguageRec.NOT_FOUND && langRec.lang == null) { + if (langRec.isRepeatFailedLangFile()) { + throw new LanguageNotFoundException(languageId, langRec.th); } + langRec.loadLanguage(monitor); } - return lang; + return langRec.lang; } - void unloadLanguage(LanguageID languageID) { - if (languages.containsKey(languageID)) { - languages.put(languageID, null); + + /** + * Returns the {@link SleighLanguageDescription language description} of the specified language + * + * @param languageId {@link LanguageID} + * @return {@link SleighLanguageDescription} + */ + public SleighLanguageDescription getLanguageDescription(LanguageID languageId) { + return languages.getOrDefault(languageId, LanguageRec.NOT_FOUND).langDesc; + } + + void unloadLanguage(LanguageID languageId) { + LanguageRec langRec = languages.get(languageId); + if (langRec != null) { + langRec.unloadLanguage(); } } @Override public LanguageDescription[] getLanguageDescriptions() { - LanguageDescription[] d = new LanguageDescription[descriptions.size()]; - descriptions.values().toArray(d); - return d; + return languages.values() + .stream() + .map(langRec -> langRec.langDesc) + .toArray(LanguageDescription[]::new); } - private void createLanguageDescriptions(final ResourceFile specFile) + private void createLanguageDescriptions(ResourceFile specFile) throws SAXException, IOException { - ErrorHandler errHandler = new ErrorHandler() { - @Override - public void error(SAXParseException exception) throws SAXException { - Msg.error(SleighLanguageProvider.this, "Error parsing " + specFile, exception); - } - - @Override - public void fatalError(SAXParseException exception) throws SAXException { - Msg.error(SleighLanguageProvider.this, "Fatal error parsing " + specFile, - exception); - } - - @Override - public void warning(SAXParseException exception) throws SAXException { - Msg.warn(SleighLanguageProvider.this, "Warning parsing " + specFile, exception); - } - }; - XmlPullParser parser = XmlPullParserFactory.create(specFile, errHandler, false); + XmlPullParser parser = + XmlPullParserFactory.create(specFile, loggingErrorHandler(specFile), false); try { read(parser, specFile.getParentFile(), specFile.getName()); } @@ -294,17 +260,12 @@ public class SleighLanguageProvider implements LanguageProvider { } while ((compiler = parser.softStart("compiler")) != null) { String compilerID = compiler.getAttribute("id"); - final CompilerSpecID compilerSpecID = new CompilerSpecID(compilerID); - final String compilerSpecName = compiler.getAttribute("name"); - final String compilerSpecFilename = compiler.getAttribute("spec"); - final ResourceFile compilerSpecFile = - findFile(parentDirectory, compilerSpecFilename, ".cspec"); - FileResolutionResult result = existsAndIsCaseDependent(compilerSpecFile); - if (!result.isOk()) { - throw new SleighException("cspec file " + compilerSpecFile + - " is not properly case dependent: " + result.getMessage()); - } - final SleighCompilerSpecDescription sleighCompilerSpecDescription = + CompilerSpecID compilerSpecID = new CompilerSpecID(compilerID); + String compilerSpecName = compiler.getAttribute("name"); + String compilerSpecFilename = compiler.getAttribute("spec"); + ResourceFile compilerSpecFile = SleighLanguageFile + .getLanguageResourceFile(parentDirectory, compilerSpecFilename, ".cspec"); + SleighCompilerSpecDescription sleighCompilerSpecDescription = new SleighCompilerSpecDescription(compilerSpecID, compilerSpecName, compilerSpecFile); compilerSpecs.add(sleighCompilerSpecDescription); @@ -343,58 +304,26 @@ public class SleighLanguageProvider implements LanguageProvider { " is not properly case dependent: " + result.getMessage()); } description.setDefsFile(defsFile); - final ResourceFile specFile = findFile(parentDirectory, pspec, ".pspec"); - result = existsAndIsCaseDependent(specFile); - if (!result.isOk()) { - throw new SleighException("pspec file " + specFile + - " is not properly case dependent: " + result.getMessage()); - } + + ResourceFile specFile = + SleighLanguageFile.getLanguageResourceFile(parentDirectory, pspec, ".pspec"); description.setSpecFile(specFile); - ResourceFile slaFile; - try { - slaFile = findFile(parentDirectory, slafilename, ".slaspec"); - result = existsAndIsCaseDependent(slaFile); - if (!result.isOk()) { - throw new SleighException("sla file " + slaFile + - " is not properly case dependent: " + result.getMessage()); - } - description.setSlaFile(slaFile); - } - catch (SleighException e) { - int index = slafilename.lastIndexOf('.'); - String slabase = slafilename.substring(0, index); - - String slaspecfilename = slabase + ".slaspec"; - - ResourceFile slaspecFile = findFile(parentDirectory, slaspecfilename, ".slaspec"); - result = existsAndIsCaseDependent(slaspecFile); - if (!result.isOk()) { - throw new SleighException("sla file source " + slaspecFile + - " is not properly case dependent: " + result.getMessage()); - } - - slaFile = new ResourceFile(slaspecFile.getParentFile(), slafilename); - description.setSlaFile(slaFile); - } + SleighLanguageFile langFile = + SleighLanguageFile.fromSlaFilename(parentDirectory, slafilename); + description.setLanguageFile(langFile); try { if (manualindexfile != null) { - ResourceFile manualIndexFile = - findFile(parentDirectory, manualindexfile, ".idx"); - result = existsAndIsCaseDependent(manualIndexFile); - if (result.isOk()) { - description.setManualIndexFile(manualIndexFile); - } - else { - throw new SleighException(result.getMessage()); - } + ResourceFile manualIndexFile = SleighLanguageFile + .getLanguageResourceFile(parentDirectory, manualindexfile, ".idx"); + description.setManualIndexFile(manualIndexFile); } } catch (SleighException ex) { // Error with the manual shouldn't prevent language from loading Msg.error(this, ex.getMessage()); } - if (descriptions.put(id, description) != null) { + if (languages.put(id, new LanguageRec(description)) != null) { Msg.showError(this, null, "Duplicate Sleigh Language ID", "Language " + id + " previously defined: " + defsFile); } @@ -402,48 +331,155 @@ public class SleighLanguageProvider implements LanguageProvider { parser.end(start); } - private ResourceFile findFile(ResourceFile parentDir, String fileNameOrRelativePath, - String extension) throws SleighException { - ResourceFile file = new ResourceFile(parentDir, fileNameOrRelativePath); - if (file.exists()) { - return file; - } - String fileName = getFileNameFromPath(fileNameOrRelativePath); - List files = findFiles(fileName, extension); - if (files.size() == 1) { - return files.get(0); - } + //--------------------------------------------------------------------------------------------- + /** + * Timeout used when trying to acquire the sla file lock. (Default: 60 seconds) + *

+ * The sla lock is acquired and held during .sla file checking and writing (compiling). + * Currently parsing the sla xml is done after releasing the lock. + *

+ * If a Ghidra process is trying to fetch a sleigh language, which requires acquiring the lock + * on a sla file, and it times out before succeeding, the caller will get a + * {@link LanguageNotFoundException} exception. + *

+ * This timeout should be long enough to allow the process that has the lock to finish + * compiling and writing the slaspec so that the waiting process does not give up too quickly + * and give an error to the user. + * + * See {@link #LANGUAGE_LOCK_TIMEOUT_PROPNAME}. + */ + public static final Duration LANGUAGE_LOCK_TIMEOUT = getLanguageLockTimeout(); - String relativePath = discardRelativePath(fileNameOrRelativePath); - for (ResourceFile resourceFile : files) { - if (file.getAbsolutePath().endsWith(relativePath)) { - return resourceFile; + private static final int DEFAULT_LOCK_TIMEOUT_SECS = 60; + + private static Duration getLanguageLockTimeout() { + String langLockTimeoutOverride = System.getProperty(LANGUAGE_LOCK_TIMEOUT_PROPNAME); + if (langLockTimeoutOverride != null) { + try { + return Duration.ofMillis(Long.parseLong(langLockTimeoutOverride)); + } + catch (NumberFormatException e) { + // fallthru } } - ResourceFile missingFile = new ResourceFile(parentDir, fileNameOrRelativePath); - throw new SleighException("Missing sleigh file: " + missingFile.getAbsolutePath()); + return Duration.ofSeconds(DEFAULT_LOCK_TIMEOUT_SECS); } - private String discardRelativePath(String str) { - return RELATIVE_PATHS_PATTERN.matcher(str).replaceFirst(""); - } + private static class LanguageRec { + static final LanguageRec NOT_FOUND = new LanguageRec(null); - private List findFiles(String fileName, String extension) { - List matches = new ArrayList(); - List files = Application.findFilesByExtensionInApplication(extension); - for (ResourceFile resourceFile : files) { - if (resourceFile.getName().equals(fileName)) { - matches.add(resourceFile); + SleighLanguage lang; + SleighLanguageDescription langDesc; + Long badSlaspecTS; + Throwable th; + + LanguageRec(SleighLanguageDescription langDesc) { + this.langDesc = langDesc; + } + + void markLangFileFailed(Throwable e) { + ResourceFile slaSpecFile = langDesc.getLanguageFile().getSlaSpecFile(); + badSlaspecTS = slaSpecFile.lastModified(); + th = e; + } + + boolean isRepeatFailedLangFile() { + if (th instanceof SleighFileLockException) { + // allow user to retry getting langauge because the exception would have had a built-in delay + return false; + } + // Prevents trying to load the same language over and over again, but still allows the + // user/developer to fix a slaspec file (and change its timestamp) and try to load it + // again without having to restart ghidra. Does not consider timestamps of sinc files. + if (badSlaspecTS != null) { + ResourceFile slaSpecFile = langDesc.getLanguageFile().getSlaSpecFile(); + long slaSpecTS = slaSpecFile.lastModified(); + if (slaSpecTS != badSlaspecTS) { + badSlaspecTS = null; + } + } + return badSlaspecTS != null; + } + + /** + * Loads the language. + * + * @param monitor {@link TaskMonitor}. See + * {@link SleighLanguage#SleighLanguage(SleighLanguageDescription, TaskMonitor)} for notes + * about how the TaskMonitor's settings are modified during loading. + * @throws LanguageNotFoundException if error reading language info + */ + void loadLanguage(TaskMonitor monitor) throws LanguageNotFoundException { + try { + lang = new SleighLanguage(langDesc, monitor); + badSlaspecTS = null; + } + catch (SleighException e) { + markLangFileFailed(e); + if (!(e instanceof SleighFileException)) { + // don't force showing error if its just a missing file because it will be displayed elsewhere + Msg.showError(this, null, "Error", + "Failed to read language %s".formatted(langDesc.getLanguageFile()), e); + } + throw new LanguageNotFoundException(langDesc.getLanguageID(), e); } } - return matches; + + void unloadLanguage() { + lang = null; + badSlaspecTS = null; + th = null; + } } - private String getFileNameFromPath(String fileNameOrRelativePath) { - int lastIndexOf = fileNameOrRelativePath.lastIndexOf("/"); - if (lastIndexOf < 0) { - return fileNameOrRelativePath; - } - return fileNameOrRelativePath.substring(lastIndexOf + 1); + /** + * Creates a SAX ErrorHandler that re-throws the exceptions, ignores the warning + * + * @return new {@link ErrorHandler} + */ + static ErrorHandler throwingErrorHandler() { + return new ErrorHandler() { + @Override + public void warning(SAXParseException exception) throws SAXException { + // ignore + } + + @Override + public void fatalError(SAXParseException exception) throws SAXException { + throw exception; + } + + @Override + public void error(SAXParseException exception) throws SAXException { + throw exception; + } + }; } + + /** + * Creates a SAX ErrorHandler that logs the exceptions + * + * @param specFile the input file + * @return new {@link ErrorHandler} + */ + static ErrorHandler loggingErrorHandler(ResourceFile specFile) { + return new ErrorHandler() { + @Override + public void error(SAXParseException exception) throws SAXException { + Msg.error(SleighLanguageProvider.class, "Error parsing " + specFile, exception); + } + + @Override + public void fatalError(SAXParseException exception) throws SAXException { + Msg.error(SleighLanguageProvider.class, "Fatal error parsing " + specFile, + exception); + } + + @Override + public void warning(SAXParseException exception) throws SAXException { + Msg.warn(SleighLanguageProvider.class, "Warning parsing " + specFile, exception); + } + }; + } + } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java index 742341daa6..0e3a299292 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcode/utils/SlaFormat.java @@ -200,18 +200,26 @@ public class SlaFormat { * @throws IOException for any errors reading from the stream */ public static boolean isSlaFormat(InputStream stream) throws IOException { + return getSlaFormat(stream) == FORMAT_VERSION; + } + + /** + * Returns the format version number of the specified binary .sla file. + * + * @param stream {@link InputStream} + * @return sla version number, -1 if invalid header + * @throws IOException if error reading + */ + public static int getSlaFormat(InputStream stream) throws IOException { byte[] header = new byte[4]; int readLen = stream.read(header); if (readLen < 4) { - return false; + return -1; } if (header[0] != 's' || header[1] != 'l' || header[2] != 'a') { - return false; + return -1; } - if (header[3] != FORMAT_VERSION) { - return false; - } - return true; + return Byte.toUnsignedInt(header[3]); } /** diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java index 9b63f8172b..295fbbc141 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java @@ -1784,6 +1784,22 @@ public class SleighCompile extends SleighBase { return 0; } + public void setOptions(SleighCompileOptions options) { + Set> entrySet = options.preprocs.entrySet(); + for (Entry entry : entrySet) { + setPreprocValue(entry.getKey(), entry.getValue()); + } + setUnnecessaryPcodeWarning(options.unnecessaryPcodeWarning); + setLenientConflict(options.lenientConflict); + setLocalCollisionWarning(options.allCollisionWarning); + setAllNopWarning(options.allNopWarning); + setDeadTempWarning(options.deadTempWarning); + setUnusedFieldWarning(options.unusedFieldWarning); + setEnforceLocalKeyWord(options.enforceLocalKeyWord); + setInsensitiveDuplicateError(!options.caseSensitiveRegisterNames); + setDebugOutput(options.debugOutput); + } + public void setAllOptions(Map preprocs, boolean unnecessaryPcodeWarning, boolean lenientConflict, boolean allCollisionWarning, boolean allNopWarning, boolean deadTempWarning, boolean unusedFieldWarning, boolean enforceLocalKeyWord, diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java index 14223ccc3d..6b034433ad 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileLauncher.java @@ -16,18 +16,16 @@ package ghidra.pcodeCPort.slgh_compile; import java.io.*; -import java.util.*; +import java.util.TreeSet; import org.antlr.runtime.RecognitionException; import org.jdom2.JDOMException; -import generic.jar.ResourceFile; import ghidra.GhidraApplicationLayout; import ghidra.GhidraLaunchable; +import ghidra.app.plugin.processors.sleigh.SleighException; import ghidra.framework.Application; import ghidra.framework.ApplicationConfiguration; -import ghidra.util.Msg; -import ghidra.util.SystemUtilities; /** * SleighCompileLauncher Sleigh compiler launch provider @@ -59,250 +57,68 @@ public class SleighCompileLauncher implements GhidraLaunchable { * @throws RecognitionException for parse errors */ public static int runMain(String[] args) throws IOException, RecognitionException { - int retval; - String filein = null; - String fileout = null; - Map preprocs = new HashMap<>(); - - SleighCompile.yydebug = false; - boolean allMode = false; - - if (args.length < 1) { - // @formatter:off - Msg.info(SleighCompile.class, "Usage: sleigh [options...] [ [] | -a ]"); - Msg.info(SleighCompile.class, " sleigh [options...] []"); - Msg.info(SleighCompile.class, " source slaspec file to be compiled"); - Msg.info(SleighCompile.class, " optional output sla file (infile.sla assumed)"); - Msg.info(SleighCompile.class, " or"); - Msg.info(SleighCompile.class, " sleigh [options...] -a "); - Msg.info(SleighCompile.class, " directory to have all slaspec files compiled"); - Msg.info(SleighCompile.class, " options:"); - Msg.info(SleighCompile.class, " -x turns on parser debugging"); - Msg.info(SleighCompile.class, " -y write .sla using XML debug format"); - Msg.info(SleighCompile.class, " -u print warnings for unnecessary pcode instructions"); - Msg.info(SleighCompile.class, " -l report pattern conflicts"); - Msg.info(SleighCompile.class, " -n print warnings for all NOP constructors"); - Msg.info(SleighCompile.class, " -t print warnings for dead temporaries"); - Msg.info(SleighCompile.class, " -e enforce use of 'local' keyword for temporaries"); - Msg.info(SleighCompile.class, " -c print warnings for all constructors with colliding operands"); - Msg.info(SleighCompile.class, " -f print warnings for unused token fields"); - Msg.info(SleighCompile.class, " -s treat register names as case sensitive"); - Msg.info(SleighCompile.class, " -DNAME=VALUE defines a preprocessor macro NAME with value VALUE (option may be repeated)"); - Msg.info(SleighCompile.class, " -dMODULE defines a preprocessor macro MODULE with a value of its module path (option may be repeated)"); - Msg.info(SleighCompile.class, " -i inject options from specified file"); - // @formatter:on + if (args.length == 0) { + SleighCompileOptions.usage(); return 2; } + SleighCompileOptions options; + try { + options = SleighCompileOptions.parse(args); + } + catch (SleighException e) { + return 1; + } + return launchCompile(options); + } - boolean unnecessaryPcodeWarning = false; - boolean lenientConflict = true; - boolean allCollisionWarning = false; - boolean allNopWarning = false; - boolean deadTempWarning = false; - boolean enforceLocalKeyWord = false; - boolean unusedFieldWarning = false; - boolean caseSensitiveRegisterNames = false; - boolean debugOutput = false; + public static int launchCompile(SleighCompileOptions options) + throws IOException, RecognitionException { + if (options.allMode) { + return compileAll(options); + } + else { + return compileOne(options); + } + } - int i; - for (i = 0; i < args.length; ++i) { - if (args[i].charAt(0) != '-') { - break; - } - else if (args[i].charAt(1) == 'i') { - // inject options from file specified by next argument - args = injectOptionsFromFile(args, ++i); - if (args == null) { - return 1; - } - } - else if (args[i].charAt(1) == 'D') { - String preproc = args[i].substring(2); - int pos = preproc.indexOf('='); - if (pos == -1) { - Msg.error(SleighCompile.class, "Bad sleigh option: " + args[i]); - return 1; - } - String name = preproc.substring(0, pos); - String value = preproc.substring(pos + 1); - preprocs.put(name, value); // Preprocessor macro definitions - } - else if (args[i].charAt(1) == 'd') { - String moduleName = args[i].substring(2); - ResourceFile module = Application.getModuleRootDir(moduleName); - if (module == null || !module.isDirectory()) { - Msg.error(SleighCompile.class, - "Failed to resolve module reference: " + args[i]); - return 1; - } - Msg.debug(SleighCompile.class, - "Sleigh resolved module: " + moduleName + "=" + module.getAbsolutePath()); - preprocs.put(moduleName, module.getAbsolutePath()); // Preprocessor macro definitions - } - else if (args[i].charAt(1) == 'u') { - unnecessaryPcodeWarning = true; - } - else if (args[i].charAt(1) == 't') { - deadTempWarning = true; - } - else if (args[i].charAt(1) == 'e') { - enforceLocalKeyWord = true; - } - else if (args[i].charAt(1) == 'f') { - unusedFieldWarning = true; - } - else if (args[i].charAt(1) == 'l') { - lenientConflict = false; - } - else if (args[i].charAt(1) == 'c') { - allCollisionWarning = true; - } - else if (args[i].charAt(1) == 'n') { - allNopWarning = true; - } - else if (args[i].charAt(1) == 'a') { - allMode = true; - } - else if (args[i].charAt(1) == 's') { - caseSensitiveRegisterNames = true; - } - else if (args[i].charAt(1) == 'y') { - debugOutput = true; - } - else if (args[i].charAt(1) == 'x') { - SleighCompile.yydebug = true; // Debug option + public static int compileOne(SleighCompileOptions options) + throws IOException, RecognitionException { + SleighCompile compiler = new SleighCompile(); + compiler.setOptions(options); + + return compiler.run_compilation(options.inputFile.getPath(), options.outputFile.getPath()); + } + + public static int compileAll(SleighCompileOptions options) + throws IOException, RecognitionException { + TreeSet failures = new TreeSet<>(); + int totalFailures = 0; + int totalSuccesses = 0; + DirectoryVisitor visitor = new DirectoryVisitor(options.allDir, SLASPEC_FILTER); + for (File input : visitor) { + System.out.println("Compiling " + input + ":"); + SleighCompile compiler = new SleighCompile(); + compiler.setOptions(options); + + String outname = input.getName().replace(".slaspec", ".sla"); + File output = new File(input.getParentFile(), outname); + int retval = compiler.run_compilation(input.getPath(), output.getPath()); + System.out.println(); + if (retval != 0) { + ++totalFailures; + failures.add(input.getPath()); } else { - Msg.error(SleighCompile.class, "Unknown option: " + args[i]); - return 1; + ++totalSuccesses; } } - - if (i < args.length - 2) { - Msg.error(SleighCompile.class, "Too many parameters"); - return 1; - } - - if (allMode) { - if (i == args.length) { - Msg.error(SleighCompile.class, "Missing input directory path"); - return 1; + System.out.println(totalSuccesses + " languages successfully compiled"); + if (totalFailures != 0) { + for (String path : failures) { + System.out.println(path + " failed to compile"); } - String directory = args[i]; - File dir = new File(directory); - if (!dir.exists() || !dir.isDirectory()) { - Msg.error(SleighCompile.class, directory + " is not a directory"); - return 1; - } - TreeSet failures = new TreeSet<>(); - int totalFailures = 0; - int totalSuccesses = 0; - DirectoryVisitor visitor = new DirectoryVisitor(dir, SLASPEC_FILTER); - for (File input : visitor) { - System.out.println("Compiling " + input + ":"); - SleighCompile compiler = new SleighCompile(); - compiler.setAllOptions(preprocs, unnecessaryPcodeWarning, lenientConflict, - allCollisionWarning, allNopWarning, deadTempWarning, unusedFieldWarning, - enforceLocalKeyWord, caseSensitiveRegisterNames, - debugOutput); - - String outname = input.getName().replace(".slaspec", ".sla"); - File output = new File(input.getParent(), outname); - retval = - compiler.run_compilation(input.getAbsolutePath(), output.getAbsolutePath()); - System.out.println(); - if (retval != 0) { - ++totalFailures; - failures.add(input.getAbsolutePath()); - } - else { - ++totalSuccesses; - } - } - System.out.println(totalSuccesses + " languages successfully compiled"); - if (totalFailures != 0) { - for (String path : failures) { - System.out.println(path + " failed to compile"); - } - System.out.println(totalFailures + " languages total failed to compile"); - } - return -totalFailures; + System.out.println(totalFailures + " languages total failed to compile"); } - - // single file compile - SleighCompile compiler = new SleighCompile(); - compiler.setAllOptions(preprocs, unnecessaryPcodeWarning, lenientConflict, - allCollisionWarning, allNopWarning, deadTempWarning, unusedFieldWarning, - enforceLocalKeyWord, caseSensitiveRegisterNames, debugOutput); - if (i == args.length) { - Msg.error(SleighCompile.class, "Missing input file name"); - return 1; - } - - filein = args[i]; - if (i < args.length - 1) { - fileout = args[i + 1]; - } - - String baseName = filein; - if (filein.toLowerCase().endsWith(FILE_IN_DEFAULT_EXT)) { - baseName = filein.substring(0, filein.length() - FILE_IN_DEFAULT_EXT.length()); - } - filein = baseName + FILE_IN_DEFAULT_EXT; - - String baseOutName = fileout; - if (fileout == null) { - baseOutName = baseName; - } - else if (fileout.toLowerCase().endsWith(FILE_OUT_DEFAULT_EXT)) { - baseOutName = fileout.substring(0, fileout.length() - FILE_OUT_DEFAULT_EXT.length()); - } - fileout = baseOutName + FILE_OUT_DEFAULT_EXT; - - return compiler.run_compilation(filein, fileout); + return -totalFailures; } - - private static String[] injectOptionsFromFile(String[] args, int index) { - if (index >= args.length) { - Msg.error(SleighCompile.class, "Missing options input file name"); - return null; - } - - File optionsFile = new File(args[index]); - if (!optionsFile.isFile()) { - Msg.error(SleighCompile.class, - "Options file not found: " + optionsFile.getAbsolutePath()); - if (SystemUtilities.isInDevelopmentMode()) { - Msg.error(SleighCompile.class, - "Eclipse language module must be selected and 'gradle prepdev' prevously run"); - } - return null; - } - ArrayList list = new ArrayList<>(); - for (int i = 0; i <= index; i++) { - list.add(args[i]); - } - - try (BufferedReader r = new BufferedReader(new FileReader(optionsFile))) { - String option = r.readLine(); - while (option != null) { - option = option.trim(); - if (option.length() != 0 && !option.startsWith("#")) { - list.add(option); - } - option = r.readLine(); - } - } - catch (IOException e) { - Msg.error(SleighCompile.class, - "Reading options file failed (" + optionsFile.getName() + "): " + e.getMessage()); - return null; - } - - for (int i = index + 1; i < args.length; i++) { - list.add(args[i]); - } - return list.toArray(new String[list.size()]); - } - } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptions.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptions.java new file mode 100644 index 0000000000..78543ca6b2 --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptions.java @@ -0,0 +1,283 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.pcodeCPort.slgh_compile; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +import org.apache.commons.io.FilenameUtils; + +import generic.jar.ResourceFile; +import ghidra.app.plugin.processors.sleigh.SleighException; +import ghidra.app.plugin.processors.sleigh.SleighLanguageFile; +import ghidra.framework.Application; +import ghidra.util.Msg; +import ghidra.util.SystemUtilities; +import utilities.util.FileUtilities; + +/** + * Represents the options the sleigh compiler uses + */ +public class SleighCompileOptions { + public File inputFile; + public File outputFile; + public boolean allMode = false; + public File allDir; + + public Map preprocs = new HashMap<>(); + public boolean unnecessaryPcodeWarning = false; + public boolean lenientConflict = true; + public boolean allCollisionWarning = false; + public boolean allNopWarning = false; + public boolean deadTempWarning = false; + public boolean enforceLocalKeyWord = false; + public boolean unusedFieldWarning = false; + public boolean caseSensitiveRegisterNames = false; + public boolean debugOutput = false; + + public static void usage() { + // @formatter:off + Msg.info(SleighCompileOptions.class, "Usage: sleigh [options...] [ [] | -a ]"); + Msg.info(SleighCompileOptions.class, " sleigh [options...] []"); + Msg.info(SleighCompileOptions.class, " source slaspec file to be compiled"); + Msg.info(SleighCompileOptions.class, " optional output sla file (infile.sla assumed)"); + Msg.info(SleighCompileOptions.class, " or"); + Msg.info(SleighCompileOptions.class, " sleigh [options...] -a "); + Msg.info(SleighCompileOptions.class, " directory to have all slaspec files compiled"); + Msg.info(SleighCompileOptions.class, " options:"); + Msg.info(SleighCompileOptions.class, " -x turns on parser debugging"); + Msg.info(SleighCompileOptions.class, " -y write .sla using XML debug format"); + Msg.info(SleighCompileOptions.class, " -u print warnings for unnecessary pcode instructions"); + Msg.info(SleighCompileOptions.class, " -l report pattern conflicts"); + Msg.info(SleighCompileOptions.class, " -n print warnings for all NOP constructors"); + Msg.info(SleighCompileOptions.class, " -t print warnings for dead temporaries"); + Msg.info(SleighCompileOptions.class, " -e enforce use of 'local' keyword for temporaries"); + Msg.info(SleighCompileOptions.class, " -c print warnings for all constructors with colliding operands"); + Msg.info(SleighCompileOptions.class, " -f print warnings for unused token fields"); + Msg.info(SleighCompileOptions.class, " -s treat register names as case sensitive"); + Msg.info(SleighCompileOptions.class, " -DNAME=VALUE defines a preprocessor macro NAME with value VALUE (option may be repeated)"); + Msg.info(SleighCompileOptions.class, " -dMODULE defines a preprocessor macro MODULE with a value of its module path (option may be repeated)"); + Msg.info(SleighCompileOptions.class, " -i inject options from specified file"); + // @formatter:on + } + + /** + * Evaluates an array of string arguments and saves the values into a {@link SleighCompileOptions} + * instance. + * + * @param args array of arg strings + * @return new {@link SleighCompileOptions} instance + * @throws SleighException if error in an argument + */ + public static SleighCompileOptions parse(String[] args) throws SleighException { + + SleighCompileOptions results = new SleighCompileOptions(); + + Deque argList = new ArrayDeque<>(List.of(args)); + while (!argList.isEmpty()) { + if (!argList.peekFirst().startsWith("-")) { + break; + } + String arg = argList.removeFirst(); + if (arg.isBlank()) { + continue; + } + results.processArg(arg, argList); + } + + if (!results.allMode) { + // Process trailing source and destination filename parameters + if (argList.isEmpty()) { + Msg.error(SleighCompileOptions.class, "Missing input file name"); + throw new SleighException("Missing input file name"); + } + + results.inputFile = new File(argList.removeFirst()).getAbsoluteFile(); + results.outputFile = + !argList.isEmpty() ? new File(argList.removeFirst()).getAbsoluteFile() : null; + + String baseName = results.inputFile.getName(); + if (baseName.toLowerCase().endsWith(SleighLanguageFile.SLASPEC_EXT)) { + baseName = FilenameUtils.getBaseName(baseName); + } + else { + results.inputFile = new File(results.inputFile.getParentFile(), + results.inputFile.getName() + SleighLanguageFile.SLASPEC_EXT); + } + + if (results.outputFile == null) { + results.outputFile = new File(results.inputFile.getParentFile(), + baseName + SleighLanguageFile.SLA_EXT); + } + else if (!results.outputFile.getName() + .toLowerCase() + .endsWith(SleighLanguageFile.SLA_EXT)) { + results.outputFile = new File(results.outputFile.getParentFile(), + results.outputFile.getName() + SleighLanguageFile.SLA_EXT); + } + } + + if (!argList.isEmpty()) { + Msg.error(SleighCompileOptions.class, "Too many parameters: " + argList.toString()); + throw new SleighException("Too many parameters: " + argList.toString()); + } + + + return results; + } + + /** + * Evaluates the arguments in a sleighArgs.txt file and returns a {@link SleighCompileOptions} + * + * @param argsFile file containing an argument per line + * @return {@link SleighCompileOptions} + */ + public static SleighCompileOptions fromFile(File argsFile) { + SleighCompileOptions results = new SleighCompileOptions(); + + Deque argList = new ArrayDeque<>(getOptionsFromFile(argsFile)); + while (!argList.isEmpty()) { + String arg = argList.removeFirst(); + if (arg.isBlank()) { + continue; + } + if (!arg.startsWith("-")) { + break; + } + results.processArg(arg, argList); + } + return results; + } + + public void addPreprocessorMacroDefinition(String name, String value) { + preprocs.put(name, value); // Preprocessor macro definitions + } + + private void processArg(String arg, Deque argList) { + if (arg.length() < 2 || arg.charAt(0) != '-') { + throw new SleighException("Invalid argument: " + arg); + } + switch (arg.charAt(1)) { + case 'i': + // inject options from file specified by next argument + if (argList.isEmpty()) { + Msg.error(SleighCompileOptions.class, "Missing options input file name"); + throw new SleighException("Missing options input file name"); + } + String optFilename = argList.removeFirst(); + File optionsFile = new File(optFilename).getAbsoluteFile(); + if (!optionsFile.isFile()) { + Msg.error(SleighCompileOptions.class, "Options file not found: " + optionsFile); + if (SystemUtilities.isInDevelopmentMode()) { + Msg.error(SleighCompileOptions.class, + "Eclipse language module must be selected and 'gradle prepdev' prevously run"); + } + throw new SleighException("Options file not found: " + optionsFile); + } + Deque newArgList = new ArrayDeque<>(getOptionsFromFile(optionsFile)); + newArgList.addAll(argList); + argList.clear(); + argList.addAll(newArgList); + break; + case 'D': + String preproc = arg.substring(2); + int pos = preproc.indexOf('='); + if (pos == -1) { + Msg.error(SleighCompileOptions.class, "Bad sleigh option: " + arg); + throw new SleighException("Bad sleigh option: " + arg); + } + String name = preproc.substring(0, pos); + String value = preproc.substring(pos + 1); + addPreprocessorMacroDefinition(name, value); + break; + case 'd': + String moduleName = arg.substring(2); + ResourceFile module = Application.getModuleRootDir(moduleName); + if (module == null || !module.isDirectory()) { + Msg.error(SleighCompileOptions.class, + "Failed to resolve module reference: " + arg); + throw new SleighException("Failed to resolve module reference: " + arg); + } + Msg.debug(SleighCompileOptions.class, + "Sleigh resolved module: " + moduleName + "=" + module.getAbsolutePath()); + addPreprocessorMacroDefinition(moduleName, module.getAbsolutePath()); + break; + case 'u': + unnecessaryPcodeWarning = true; + break; + case 't': + deadTempWarning = true; + break; + case 'e': + enforceLocalKeyWord = true; + break; + case 'f': + unusedFieldWarning = true; + break; + case 'l': + lenientConflict = false; + break; + case 'c': + allCollisionWarning = true; + break; + case 'n': + allNopWarning = true; + break; + case 'a': + if (argList.isEmpty()) { + Msg.error(SleighCompileOptions.class, "Missing input directory path"); + throw new SleighException("Missing input directory path"); + } + File dir = new File(argList.removeFirst()).getAbsoluteFile(); + if (!dir.exists() || !dir.isDirectory()) { + Msg.error(SleighCompileOptions.class, dir + " is not a directory"); + throw new SleighException(dir + " is not a directory"); + } + allMode = true; + allDir = dir; + break; + case 's': + caseSensitiveRegisterNames = true; + break; + case 'y': + debugOutput = true; + break; + case 'x': + SleighCompile.yydebug = true; // Debug option + break; + default: + Msg.error(SleighCompileOptions.class, "Unknown option: " + arg); + throw new SleighException("Unknown option: " + arg); + } + } + + private static List getOptionsFromFile(File optionsFile) throws SleighException { + try { + return FileUtilities.getLines(optionsFile) + .stream() + .map(String::trim) + .filter(option -> !option.isBlank() && !option.startsWith("#")) + .toList(); + } + catch (IOException e) { + Msg.error(SleighCompileOptions.class, + "Reading options file failed (" + optionsFile.getName() + "): " + e.getMessage()); + throw new SleighException("Failed reading options file [%s]".formatted(optionsFile), e); + } + } + +} diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java index ca94bb00c9..94d9395813 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageNotFoundException.java @@ -1,6 +1,5 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,10 +35,21 @@ public class LanguageNotFoundException extends IOException { /** * Language not found - * @param languageID + * + * @param languageID {@link LanguageID} */ public LanguageNotFoundException(LanguageID languageID) { - super("Language not found for '" + languageID + "'"); + this(languageID, (Throwable) null); + } + + /** + * Language not found because of an exception + * + * @param languageID {@link LanguageID} + * @param cause {@link Throwable} that caused the language to not be found + */ + public LanguageNotFoundException(LanguageID languageID, Throwable cause) { + super("Language not found for '" + languageID + "'", cause); } public LanguageNotFoundException(String message) { diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java index e882e168cf..7477beabc6 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/LanguageProvider.java @@ -16,6 +16,7 @@ package ghidra.program.model.lang; import ghidra.util.classfinder.ExtensionPoint; +import ghidra.util.task.TaskMonitor; /** * NOTE: ALL LanguageProvider CLASSES MUST END IN "LanguageProvider". If not, @@ -31,9 +32,22 @@ public interface LanguageProvider extends ExtensionPoint { * * @param languageId the name of the language to be retrieved * @return the {@link Language} with the given name or null if not found - * @throws RuntimeException if language instantiation error occurs + * @throws LanguageNotFoundException if language instantiation error */ - Language getLanguage(LanguageID languageId); + default Language getLanguage(LanguageID languageId) throws LanguageNotFoundException { + return getLanguage(languageId, TaskMonitor.DUMMY); + } + + /** + * Returns the language with the given name or null if no language has that name. + * + * @param languageId the name of the language to be retrieved + * @param monitor {@link TaskMonitor} + * @return the {@link Language} with the given name or null if not found + * @throws LanguageNotFoundException if language instantiation error + */ + Language getLanguage(LanguageID languageId, TaskMonitor monitor) + throws LanguageNotFoundException; /** * Returns a list of language descriptions provided by this provider diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java index 13df7a0d20..7516e3a4c5 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PackedDecode.java @@ -15,8 +15,7 @@ */ package ghidra.program.model.pcode; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.util.ArrayList; import ghidra.program.model.address.AddressFactory; @@ -50,7 +49,7 @@ import ghidra.program.model.address.AddressSpace; * For strings, the integer encoded after the \e type byte, is the actual length of the string. The * string data itself is stored immediately after the length integer using UTF8 format. * */ -public class PackedDecode implements Decoder { +public class PackedDecode implements Decoder, Closeable { public static final int HEADER_MASK = 0xc0; public static final int ELEMENT_START = 0x40; @@ -231,6 +230,7 @@ public class PackedDecode implements Decoder { * Close stream cached by the ingestStreamAsNeeded method. * @throws IOException for low-level problems with the stream */ + @Override public void close() throws IOException { inStream.close(); inStream = null; diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java index 2a1b28b196..e3cdba3b98 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/DefaultLanguageService.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,6 +16,7 @@ package ghidra.program.util; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -23,6 +24,7 @@ import org.apache.logging.log4j.Logger; import ghidra.app.plugin.processors.sleigh.SleighLanguageProvider; import ghidra.program.model.lang.*; import ghidra.util.task.TaskBuilder; +import ghidra.util.task.TaskMonitor; /** * Default Language service used gather up all the languages that were found @@ -318,27 +320,54 @@ public class DefaultLanguageService implements LanguageService { this.provider = lp; } - // synchronized to prevent multiple clients from trying to load the language at once - synchronized Language getLanguage() { + /** + * Loads a {@link Language} from a {@link LanguageProvider}, using a Task with a modal + * {@link TaskMonitor}. + *

+ * This is the call path that users will see when using Ghidra and a language is + * initially loaded from disk. + * + * @return {@link Language} + * @throws LanguageNotFoundException if error initializing Language + */ + synchronized Language getLanguage() throws LanguageNotFoundException { + // synchronized to prevent multiple clients from trying to load the language at once LanguageID id = description.getLanguageID(); if (provider.isLanguageLoaded(id)) { // already loaded; no need to create a task - return provider.getLanguage(id); + return provider.getLanguage(id, TaskMonitor.DUMMY); } - //@formatter:off - TaskBuilder.withRunnable(monitor -> { - provider.getLanguage(id); // load and cache - }) - .setTitle("Loading language '" + id + "'") - .setCanCancel(false) - .setHasProgress(false) - .launchModal() - ; - //@formatter:on + AtomicReference langError = new AtomicReference<>(); + AtomicReference langResult = new AtomicReference<>(); - return provider.getLanguage(id); + //@formatter:off + // Need to start task with canCancel true to get the cancel button added to the task dialog + TaskBuilder.withRunnable(monitor -> { + monitor.setCancelEnabled(false); + try { + langResult.set(provider.getLanguage(id, monitor)); + } + catch (Throwable th) { + langError.set(th); + } + }) + .setTitle("Loading language '%s'".formatted(id)) + .setCanCancel(true) + .setHasProgress(false) + .launchModal(); + //@formatter:on + + Throwable th = langError.get(); + if (th instanceof LanguageNotFoundException lnfe) { + throw lnfe; + } + else if (th != null) { + throw new LanguageNotFoundException(id, th); + } + + return langResult.get(); } @Override diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/JavaProcessBuilder.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/JavaProcessBuilder.java new file mode 100644 index 0000000000..fd6f65d3f0 --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/JavaProcessBuilder.java @@ -0,0 +1,254 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.processors.sleigh; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.function.Consumer; + +import generic.concurrent.io.ProcessConsumer; + +/** + * Helps creating and launching a java process. + *

+ * By default, the launched process will have the same classpath as the current jvm and use the + * same java binary to start the session. + *

+ * Example usage: + * 

+ *   Process newproc = new JavaProcessBuilder(AClassWithMainEntryPoint.class)
+ *     .addProperty("sun.java2d.opengl", "false")
+ *     .addLaunchArg("-Xmx2G")
+ *     .withStdoutMonitor( (s) -> System.out.println(s) )
+ *     .start();
+ *
+ */ +class JavaProcessBuilder { + private static final String CP_SEP = System.getProperty("path.separator"); + + private File javaBinary; + private String mainClassname; + private List arguments; + private String classPaths; + private Map javaProperties = new HashMap<>(); + private List launchArgs = new ArrayList<>(); + private Consumer stdoutMonitor; + + /** + * Creates a java process builder, setting the main entry point for the launched process. + * + * @param mainClass class that contains a {@code public static void main(String[])} entry point. + * Also implies that the launched process should have the same classpath as the current + * jvm, or that the referenced class is somehow included in the launched process's classpath + */ + public JavaProcessBuilder(Class mainClass) { + this.mainClassname = mainClass.getName(); + } + + /** + * Creates a java process builder, setting the main entry point for the launched process. + * + * @param mainClassname name of a class that contains a + * {@code public static void main(String[])} entry point. + */ + public JavaProcessBuilder(String mainClassname) { + this.mainClassname = mainClassname; + } + + /** + * Sets the arguments for the launched processes {@code main(String[] args)} method. + * + * @param newArguments arguments for the launched main() entry point + * @return chainable ref to same + */ + public JavaProcessBuilder withArguments(List newArguments) { + this.arguments = new ArrayList<>(newArguments); + return this; + } + + /** + * Sets the location of the java jdk install, which controls how the bin/java[.exe] is found. + * + * @param javaHomeDir java home directory + * @return chainable ref to same + */ + public JavaProcessBuilder withJavaHome(File javaHomeDir) { + this.javaBinary = javaHomeDir != null ? javaBinaryFromJavaHome(javaHomeDir) : null; + + return this; + } + + private static File javaBinaryFromJavaHome(File javaHomeDir) { + File binDir = new File(javaHomeDir, "bin"); + return new File(binDir, "java"); + } + + /** + * Returns the location of the java binary that will be used to launch the new process. + * + * @return File pointing to the java jvm binary (java or java.exe) + */ + public File getJavaBinary() { + File f = javaBinary; + if (f == null) { + f = javaBinaryFromJavaHome(new File(System.getProperty("java.home"))); + } + return f; + } + + /** + * Sets the classpaths for the launched process. The string is expected to be delimited with + * the correct separators. + * + * @param newClassPaths classpath string for the launched process + * @return chainable ref to same + */ + public JavaProcessBuilder withClasspaths(String newClassPaths) { + this.classPaths = newClassPaths; + + return this; + } + + /** + * Returns the classpath string that will be used to launch the new process. + * + * @return string + */ + public String getClasspaths() { + String s = classPaths; + if (s == null) { + s = System.getProperty("java.class.path"); + } + return s; + } + + /** + * Adds an element to the classpath. + * + * @param classPath single classpath element + * @return chainable ref to same + */ + public JavaProcessBuilder addClasspath(String classPath) { + this.classPaths = Objects.requireNonNullElse(this.classPaths, "") + CP_SEP + classPath; + + return this; + } + + /** + * Sets a callback that will handle each line written to stdout by the launched process. + *

+ * The monitor will be called an additional time with a {@code null} value after the + * spawned process has exited and its stdout has emptied. + * + * @param newStdoutMonitor Consumer that will receive each text line written by + * the launched process to it's stdout + * @return chainable ref to same + */ + public JavaProcessBuilder withStdoutMonitor(Consumer newStdoutMonitor) { + this.stdoutMonitor = newStdoutMonitor; + + return this; + } + + /** + * Adds a java system property to the launched process (e.g. "-DpropertyName=value"). + * + * @param propertyName name of the property + * @param propertyValue value of the property + * @return chainable ref to same + */ + public JavaProcessBuilder addProperty(String propertyName, String propertyValue) { + javaProperties.put(propertyName, propertyValue); + + return this; + } + + /** + * Returns a list of properties that will be added to the launched process. + * + * @return list of property definition arguments + */ + public List getProperties() { + return javaProperties.entrySet() + .stream() + .map(entry -> "-D%s=%s".formatted(entry.getKey(), entry.getValue())) + .toList(); + } + + /** + * Adds a java launch argument (e.g. "-Xmx512M", etc) + * + * @param launchArg raw argument to pass to the java binary when launching + * @return chainable ref to same + */ + public JavaProcessBuilder addLaunchArg(String launchArg) { + launchArgs.add(launchArg); + + return this; + } + + /** + * Creates a 'real' {@link ProcessBuilder} using the information specified in this builder. + *

+ * The stdout monitor will still need to be installed into any launched process. + * + * @return {@link ProcessBuilder} + */ + public ProcessBuilder getProcessBuilder() { + Objects.requireNonNull(mainClassname); + + List commandParts = new ArrayList<>(); + commandParts.add(getJavaBinary().getPath()); + commandParts.addAll(launchArgs); + String cp = getClasspaths(); + if (!cp.isBlank()) { + commandParts.add("-cp"); + commandParts.add(cp); + } + commandParts.addAll(getProperties()); + commandParts.add(mainClassname); + commandParts.addAll(arguments != null ? arguments : List.of()); + + ProcessBuilder pb = new ProcessBuilder(commandParts); + return pb; + } + + /** + * Installs a monitor that reads the processes stdout + * + * @param process {@link Process} + */ + public void installMonitor(Process process) { + ProcessConsumer.monitorAndSignalEof(process.getInputStream(), stdoutMonitor, + "stdout[%d]".formatted(process.pid())); + } + + /** + * Launches a new java process using the information specified in this builder. + * + * @return {@link Process} + * @throws IOException if error starting process + */ + public Process start() throws IOException { + ProcessBuilder pb = getProcessBuilder(); + Process process = pb.start(); + if (stdoutMonitor != null) { + installMonitor(process); + } + return process; + } +} diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProviderTest.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProviderTest.java new file mode 100644 index 0000000000..c6a8728d95 --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/SleighLanguageProviderTest.java @@ -0,0 +1,230 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.processors.sleigh; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Test; + +import generic.jar.ResourceFile; +import generic.test.AbstractGenericTest; +import ghidra.framework.Application; +import ghidra.program.model.lang.Language; +import ghidra.program.model.lang.LanguageID; +import ghidra.util.Msg; +import ghidra.util.SystemUtilities; +import ghidra.util.task.TaskMonitor; + +public class SleighLanguageProviderTest extends AbstractGenericTest { + + private LanguageID x86LangId = new LanguageID("x86:LE:32:default"); + private ResourceFile x86LdefsFile = Application.findDataFileInAnyModule("languages/x86.ldefs"); + private SleighLanguageProvider langProvider; + + + @Test(timeout = 60000 + 5000 /* 1 minute (default lock timeout) + 5 seconds */) + public void testSlaThunderingHerds() throws Exception { + // Tests when a thundering herd of processes all try to read a sleigh file at the same time + + langProvider = new SleighLanguageProvider(x86LdefsFile); + + // Ensure the lang file exist, and then tweak the timestamp to force one of the + // spawned procs to recompile the sla + SleighLanguage lang = langProvider.getLanguage(x86LangId, TaskMonitor.DUMMY); + SleighLanguageDescription langDesc = lang.getLanguageDescription(); + SleighLanguageFile langFile = langDesc.getLanguageFile(); + File slaFile = langFile.getSlaFile().getFile(false); + File slaSpecFile = langFile.getSlaSpecFile().getFile(false); + slaFile.setLastModified(slaSpecFile.lastModified() - 60 * 1000); + + int procsToStart = 20; + long langProviderTimeout = SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT.toMillis(); + long test_start = System.currentTimeMillis(); + + // 1) start a lot of processes + // 2) wait until they initialize themselves and get to step1 (they wait also for next step) + // 3) allow all launched processes to proceed at same time + // 4) wait for all to exit with success exit code + List procs = new ArrayList<>(); + for (int procNum = 0; procNum < procsToStart; procNum++) { + procs.add(TestProc.start(procNum, langProviderTimeout)); + } + + for (TestProc testProc : procs) { + if (!testProc.waitUntilStep(1)) { + fail(testProc.msg("failed to reach step1")); + } + } + + // If the spawned processes are writing to the .sla file incorrectly and simultaneously, + // giving a small delay between each process's critical action historically help expose + // the error. + int DELAY_BETWEEN_PROCESSES = 600; + for (TestProc testProc : procs) { + testProc.sendGoahead(); + sleep(DELAY_BETWEEN_PROCESSES); + } + + int badExitCount = 0; + for (TestProc testProc : procs) { + if (!testProc.waitUntilStep(2)) { + testProc.log("exited before step 2 reached"); + badExitCount++; + } + int exitCode = testProc.proc.waitFor(); + if (exitCode != 0) { + testProc.log("exited with error: %d".formatted(exitCode)); + badExitCount++; + } + } + + Msg.info(this, "Total test time: " + (System.currentTimeMillis() - test_start)); + + assertTrue(badExitCount == 0); + } + + @Test(timeout = 10000 + 5000 /* shorterTimeout + 5 seconds */) + public void testSlaLockTimeout() throws Exception { + // Test that lock timeout successfully causes a failure + langProvider = new SleighLanguageProvider(x86LdefsFile); + + SleighLanguageDescription langDesc = langProvider.getLanguageDescription(x86LangId); + SleighLanguageFile langFile = langDesc.getLanguageFile(); + + long shorterTimeoutMS = Duration.ofSeconds(10).toMillis(); + + TestProc testProc = TestProc.start(0, shorterTimeoutMS); + assertTrue(testProc.waitUntilStep(1)); + langFile.withLock(Duration.ofMillis(10), TaskMonitor.DUMMY, () -> { + testProc.sendGoahead(); + Thread.sleep(shorterTimeoutMS + 1000); // hold lock while testProc is trying to get it, force it to fail + }); + + testProc.assertExitNum(1); + } + + public void testSlaLanguage_FromOtherProcess() throws Throwable { + // this is not a junit test entry point, but instead is what is run + // in each sub-process launched by each TestProc instance. + + langProvider = new SleighLanguageProvider(x86LdefsFile); + TestProc.waitForGoahead(); + Language lang = langProvider.getLanguage(x86LangId, TaskMonitor.DUMMY); + + // write via stdout to the parent process, which will handle logging + System.out.println("STEP 2 Got language %s".formatted(lang)); + System.out.flush(); + + System.exit(0); + } + + public static void main(String[] args) { + // this is the entry point for the launched TestProcs that will be fighting over + // a sleigh language file + try { + SleighLanguageProviderTest test = new SleighLanguageProviderTest(); + test.testSlaLanguage_FromOtherProcess(); + } + catch (Throwable th) { + // write via stdout to the parent process, which will handle logging + System.out.println("Exception " + th); + th.printStackTrace(System.out); + System.exit(1); + } + } + + /** + * Handles coordinating an external java process that will be attempting to access a common + * sleigh language file. + */ + static class TestProc { + static TestProc start(int procNum, long timeoutMS) throws IOException { + TestProc testProc = new TestProc(); + testProc.procNum = procNum; + testProc.proc = new JavaProcessBuilder(SleighLanguageProviderTest.class) // self class's main() + .addProperty(SystemUtilities.TESTING_PROPERTY, "true") + .addProperty(SleighLanguageProvider.LANGUAGE_LOCK_TIMEOUT_PROPNAME, + "" + timeoutMS) + .withStdoutMonitor(testProc::update) + .start(); + testProc.log("Started"); + return testProc; + } + + Process proc; + int procNum; + AtomicInteger stepNum = new AtomicInteger(); + AtomicBoolean stdoutMonitorDone = new AtomicBoolean(); + Thread monitorThread; + + void sendGoahead() throws IOException { + proc.getOutputStream().write('\n'); + proc.getOutputStream().flush(); + } + + static void waitForGoahead() throws IOException { + // write via stdout to the parent process, which will handle logging + System.out.println("STEP 1 Waiting for synchronization go-ahead..."); + System.out.flush(); + int b = System.in.read(); + System.out.println("Read byte: " + b); + System.out.flush(); + } + + boolean waitUntilStep(int waitForStepNum) throws InterruptedException { + while (stepNum.get() < waitForStepNum) { + if (stdoutMonitorDone.get()) { + return false; + } + Thread.sleep(200); + } + return true; + } + + void update(String s) { + if (s == null) { + stdoutMonitorDone.set(true); + return; + } + log(s); + if (s.startsWith("STEP ")) { + stepNum.set(Integer.parseInt(s.split(" ")[1])); + } + } + + void assertExitNum(int exitNum) throws InterruptedException { + assertEquals(exitNum, proc.waitFor()); // wait until helper process has finished + log("exit code: " + proc.waitFor()); + } + + String msg(String s) { + return "TestProc[%d-%d] %s".formatted(procNum, proc.pid(), s); + } + + void log(String s) { + Msg.info(this, msg(s)); + } + } + +} diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/X86SleighLanguageLockerTest.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/X86SleighLanguageLockerTest.java new file mode 100644 index 0000000000..3c75e270d0 --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/app/plugin/processors/sleigh/X86SleighLanguageLockerTest.java @@ -0,0 +1,58 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.app.plugin.processors.sleigh; + +import java.time.Duration; + +import org.junit.Test; + +import generic.jar.ResourceFile; +import ghidra.GhidraApplicationLayout; +import ghidra.framework.Application; +import ghidra.framework.ApplicationConfiguration; +import ghidra.program.model.lang.LanguageID; +import ghidra.util.task.TaskMonitor; + +public class X86SleighLanguageLockerTest { + + private LanguageID x86LangId = new LanguageID("x86:LE:32:default"); + + //@Test + public void testLockLanguageForever() throws Exception { + // Locks the language's file forever, for manually testing the GUI frontend when it tries + // to load a language. + // This test should not be enabled by default. + + // Don't use a test application config because we need to use the same user specific + // .ghidra/.ghidra-ver config directories + Application.initializeApplication(new GhidraApplicationLayout(), + new ApplicationConfiguration()); + + ResourceFile x86LdefsFile = Application.findDataFileInAnyModule("languages/x86.ldefs"); + SleighLanguageProvider langProvider = new SleighLanguageProvider(x86LdefsFile); + + SleighLanguageDescription langDesc = langProvider.getLanguageDescription(x86LangId); + SleighLanguageFile langFile = langDesc.getLanguageFile(); + // we are reading from stdin, so should use stdout to prompt the user + System.out.println("Locking lang file: " + langFile.getSlaFile()); + + langFile.withLock(Duration.ofMillis(10), TaskMonitor.DUMMY, () -> { + System.out.println("Press enter to end lock"); + System.in.read(); // if user hits enter in the console of the test, will exit + }); + } + +} diff --git a/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptionsTest.java b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptionsTest.java new file mode 100644 index 0000000000..b6936747c7 --- /dev/null +++ b/Ghidra/Framework/SoftwareModeling/src/test.slow/java/ghidra/pcodeCPort/slgh_compile/SleighCompileOptionsTest.java @@ -0,0 +1,150 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.pcodeCPort.slgh_compile; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; + +import org.junit.Test; + +import generic.test.AbstractGenericTest; +import utilities.util.FileUtilities; + +public class SleighCompileOptionsTest extends AbstractGenericTest { + @Test + public void testArgFile() throws IOException { + File argTmp = createTempFile("sleigh_args"); + FileUtilities.writeStringToFile(argTmp, """ + -x + -u + -l + -n + -t + -e + -c + -f + -s + -DTESTNAME=TESTVALUE + -dx86 + """); + + // test default -x before doing any option parsing because its a global flag + assertFalse(SleighCompile.yydebug); + + SleighCompileOptions defaultOpts = new SleighCompileOptions(); + SleighCompileOptions opts = SleighCompileOptions.fromFile(argTmp); + + // -x + assertTrue(SleighCompile.yydebug); + + // -u + assertTrue(opts.unnecessaryPcodeWarning); + assertNotEquals(defaultOpts.unnecessaryPcodeWarning, opts.unnecessaryPcodeWarning); + + // -l + assertFalse(opts.lenientConflict); + assertNotEquals(defaultOpts.lenientConflict, opts.lenientConflict); + + // -n + assertTrue(opts.allNopWarning); + assertNotEquals(defaultOpts.allNopWarning, opts.allNopWarning); + + // -t + assertTrue(opts.deadTempWarning); + assertNotEquals(defaultOpts.deadTempWarning, opts.deadTempWarning); + + // -e + assertTrue(opts.enforceLocalKeyWord); + assertNotEquals(defaultOpts.enforceLocalKeyWord, opts.enforceLocalKeyWord); + + // -c + assertTrue(opts.allCollisionWarning); + assertNotEquals(defaultOpts.allCollisionWarning, opts.allCollisionWarning); + + // -f + assertTrue(opts.unusedFieldWarning); + assertNotEquals(defaultOpts.unusedFieldWarning, opts.unusedFieldWarning); + + // -s + assertTrue(opts.caseSensitiveRegisterNames); + assertNotEquals(defaultOpts.caseSensitiveRegisterNames, opts.caseSensitiveRegisterNames); + + // -D + assertNull(defaultOpts.preprocs.get("TESTNAME")); + assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME")); + + // -d + assertNull(defaultOpts.preprocs.get("x86")); + assertNotNull(opts.preprocs.get("x86")); + assertTrue(new File(opts.preprocs.get("x86")).isDirectory()); + } + + @Test + public void testCmdLineArgs_1input() throws IOException { + File argTmp = createTempFile("sleigh_args"); + FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE"); + + String[] args = { "-i", argTmp.getPath(), "inputfile.slaspec" }; + SleighCompileOptions opts = SleighCompileOptions.parse(args); + + assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME")); + assertEquals(opts.inputFile.getName(), "inputfile.slaspec"); + assertEquals(opts.outputFile.getName(), "inputfile.sla"); + } + + @Test + public void testCmdLineArgs_1input_default_ext() throws IOException { + File argTmp = createTempFile("sleigh_args"); + FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE"); + + String[] args = { "-i", argTmp.getPath(), "inputfile" }; + SleighCompileOptions opts = SleighCompileOptions.parse(args); + + assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME")); + assertEquals(opts.inputFile.getName(), "inputfile.slaspec"); + assertEquals(opts.outputFile.getName(), "inputfile.sla"); + } + + @Test + public void testCmdLineArgs_input_and_output() throws IOException { + File argTmp = createTempFile("sleigh_args"); + FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE"); + + String[] args = { "-i", argTmp.getPath(), "inputfile.slaspec", "outputfile.sla" }; + SleighCompileOptions opts = SleighCompileOptions.parse(args); + + assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME")); + assertEquals(opts.inputFile.getName(), "inputfile.slaspec"); + assertEquals(opts.outputFile.getName(), "outputfile.sla"); + } + + @Test + public void testCmdLineArgs_all() throws IOException { + File argTmp = createTempFile("sleigh_args"); + FileUtilities.writeStringToFile(argTmp, "-DTESTNAME=TESTVALUE"); + File allDir = createTempDirectory("alldir"); + + String[] args = { "-i", argTmp.getPath(), "-a", allDir.getPath() }; + SleighCompileOptions opts = SleighCompileOptions.parse(args); + + assertEquals("TESTVALUE", opts.preprocs.get("TESTNAME")); + assertTrue(opts.allMode); + assertEquals(opts.allDir.getPath(), allDir.getPath()); + } + +}