/*********************************************************************** THIS FILE IS AUTOMATICALLY GENERATED. DO NOT MODIFY DEVELOPER: Zihan Chen(vczh) ***********************************************************************/ #include "VlppParser.h" /*********************************************************************** .\PARSING.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { namespace tabling { using namespace definitions; using namespace analyzing; using namespace collections; using namespace regex; /*********************************************************************** ParsingGeneralParser ***********************************************************************/ ParsingGeneralParser::ParsingGeneralParser(Ptr _table) :table(_table) { } ParsingGeneralParser::~ParsingGeneralParser() { } Ptr ParsingGeneralParser::GetTable() { return table; } void ParsingGeneralParser::BeginParse() { } bool ParsingGeneralParser::Parse(ParsingState& state, ParsingTransitionProcessor& processor, collections::List>& errors) { BeginParse(); processor.Reset(); for(vint i=0;itoken==-1 || !token->completeToken) { errors.Add(Ptr(new ParsingError(token, L"Unrecognizable token: \""+WString::CopyFrom(token->reading, token->length)+L"\"."))); } } while(true) { ParsingState::TransitionResult result=ParseStep(state, errors); if(!result) { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, L"Internal error when parsing."))); return false; } else if(result.transitionType==ParsingState::TransitionResult::SkipToken) { if(state.GetCurrentTableTokenIndex()==ParsingTable::TokenFinish) { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, L"Failed to recover error when reaching the end of the input."))); return false; } else { state.SkipCurrentToken(); continue; } } else if(!processor.Run(result)) { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, L"Internal error when building the parsing tree."))); return false; } if(result.tableTokenIndex==ParsingTable::TokenFinish && !processor.GetProcessingAmbiguityBranch()) { break; } } return true; } Ptr ParsingGeneralParser::Parse(ParsingState& state, collections::List>& errors) { ParsingTreeBuilder builder; Parse(state, builder, errors); Ptr node=builder.GetNode(); if(!node) { errors.Add(Ptr(new ParsingError(L"Internal error when building the parsing tree after a succeeded parsing process."))); return 0; } return node; } Ptr ParsingGeneralParser::Parse(const WString& input, const WString& rule, collections::List>& errors, vint codeIndex) { ParsingState state(input, table, codeIndex); if(state.Reset(rule)==-1) { errors.Add(Ptr(new ParsingError(L"Rule \""+rule+L"\" does not exist."))); return 0; } return Parse(state, errors); } /*********************************************************************** ParsingStrictParser ***********************************************************************/ bool ParsingStrictParser::OnTestErrorRecoverExists() { return false; } void ParsingStrictParser::OnClearErrorRecover() { } ParsingState::TransitionResult ParsingStrictParser::OnErrorRecover(ParsingState& state, vint currentTokenIndex, collections::List>& errors) { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, (token==0?L"Error happened during parsing when reaching the end of the input.":L"Error happened during parsing.")))); return ParsingState::TransitionResult(); } ParsingStrictParser::ParsingStrictParser(Ptr _table) :ParsingGeneralParser(_table) { } ParsingStrictParser::~ParsingStrictParser() { } ParsingState::TransitionResult ParsingStrictParser::ParseStep(ParsingState& state, collections::List>& errors) { ParsingState::TransitionResult result; if (OnTestErrorRecoverExists()) { result = OnErrorRecover(state, -1, errors); } else { result = state.ReadToken(); if (result) { OnClearErrorRecover(); } else { vint currentTokenIndex = state.GetCurrentTableTokenIndex(); if (currentTokenIndex != -1) { result = OnErrorRecover(state, currentTokenIndex, errors); } } } return result; } /*********************************************************************** ParsingAutoRecoverParser ***********************************************************************/ ParsingAutoRecoverParser::RecoverFuture& ParsingAutoRecoverParser::GetRecoverFuture(vint index) { return recoverFutures[index]; } ParsingAutoRecoverParser::RecoverFuture& ParsingAutoRecoverParser::CreateRecoverFuture(vint index, vint previousIndex) { RecoverFuture* rf = 0; if (index >= recoverFutures.Count()) { CHECK_ERROR(index == recoverFutures.Count(), L"ParsingAutoRecoverParser::CreateRecoverFuture(vint, vint)#Wrong argument: index."); RecoverFuture recoverFuture; recoverFuture.future = new ParsingState::Future; index = recoverFutures.Add(recoverFuture); } rf = &GetRecoverFuture(index); rf->index = index; rf->previousIndex = previousIndex; return *rf; } bool ParsingAutoRecoverParser::OnTestErrorRecoverExists() { return recoveringFutureIndex != -1; } void ParsingAutoRecoverParser::OnClearErrorRecover() { recoveringFutureIndex = -1; } ParsingState::TransitionResult ParsingAutoRecoverParser::OnErrorRecover(ParsingState& state, vint currentTokenIndex, collections::List>& errors) { if(recoveringFutureIndex==-1) { List prioritizedTokens; prioritizedTokens.Add(ParsingTable::TokenFinish); CopyFrom( prioritizedTokens, Range(ParsingTable::UserTokenStart, table->GetTokenCount() - ParsingTable::UserTokenStart) ); prioritizedTokens.Add(ParsingTable::LeftRecursiveReduce); prioritizedTokens.Add(ParsingTable::NormalReduce); prioritizedTokens.Remove(currentTokenIndex); prioritizedTokens.Insert(0, currentTokenIndex); vint processingFutureIndex=-1; vint usedFutureCount=0; while(processingFutureIndexcurrentState==-1) continue; for (auto currentTableTokenIndex : prioritizedTokens) { vint newInsertedTokenCount = previous.insertedTokenCount; if (currentTableTokenIndex != ParsingTable::NormalReduce && currentTableTokenIndex != ParsingTable::LeftRecursiveReduce) { newInsertedTokenCount++; } if (currentTableTokenIndex != currentTokenIndex && newInsertedTokenCount > maxInsertedTokenCount) { continue; } RecoverFuture& now = CreateRecoverFuture(usedFutureCount, previous.index); now.insertedTokenCount = newInsertedTokenCount; if(state.ReadTokenInFuture(currentTableTokenIndex, previous.future, now.future, 0)) { if(currentTableTokenIndex==currentTokenIndex) { if(previous.future) { recoveringFutureIndex = previous.index; RecoverFuture* rf = &GetRecoverFuture(previous.index); while(rf->future->previous) { RecoverFuture* prf = &GetRecoverFuture(rf->previousIndex); prf->nextIndex = rf->index; prf->future->next = rf->future; rf = prf; } recoveringFutureIndex = rf->index; } else { recoveringFutureIndex = 0; } goto FOUND_ERROR_RECOVER_SOLUTION; } else { usedFutureCount++; } } } } } FOUND_ERROR_RECOVER_SOLUTION: RecoverFuture* rf = 0; if (recoveringFutureIndex != -1) { rf = &GetRecoverFuture(recoveringFutureIndex); if(rf->future->next) { recoveringFutureIndex = rf->nextIndex; } else { recoveringFutureIndex = -1; } } if(rf) { return state.RunTransition(rf->future->selectedItem, 0); } else { return ParsingState::TransitionResult(ParsingState::TransitionResult::SkipToken); } } ParsingAutoRecoverParser::ParsingAutoRecoverParser(Ptr _table, vint _maxInsertedTokenCount) :ParsingStrictParser(_table) , recoveringFutureIndex(-1) , maxInsertedTokenCount(_maxInsertedTokenCount == -1 ? 4 : _maxInsertedTokenCount) { } ParsingAutoRecoverParser::~ParsingAutoRecoverParser() { for (auto future : recoverFutures) { delete future.future; } } void ParsingAutoRecoverParser::BeginParse() { recoveringFutureIndex = -1; ParsingStrictParser::BeginParse(); } /*********************************************************************** ParsingAmbiguousParser ***********************************************************************/ ParsingAmbiguousParser::ParsingAmbiguousParser(Ptr _table) :ParsingGeneralParser(_table) ,consumedDecisionCount(0) { } ParsingAmbiguousParser::~ParsingAmbiguousParser() { } void ParsingAmbiguousParser::OnErrorRecover(ParsingState& state, vint currentTokenIndex, collections::List& futures, vint& begin, vint& end, collections::List>& errors) { begin=end; } vint ParsingAmbiguousParser::GetResolvableFutureLevels(collections::List& futures, vint begin, vint end) { if(end-begin<2) { return 1; } List resolvingFutures; for(vint i=begin;icurrentState!=second->currentState || first->reduceStateCount!=second->reduceStateCount || first->shiftStates.Count()!=second->shiftStates.Count() ) { return level; } else { for(vint j=0;jshiftStates.Count();j++) { if(first->shiftStates[j]!=second->shiftStates[j]) { return level; } } } } level++; for(vint i=0;iprevious)) { return level; } } } } vint ParsingAmbiguousParser::SearchPathForOneStep(ParsingState& state, collections::List& futures, vint& begin, vint& end, collections::List>& errors) { futures.Add(state.ExploreCreateRootFuture()); vint previousBegin = 0; vint previousEnd = 1; vint resolvableFutureLevels = 0; bool errorRecovered = false; while(true) { // keep all futures that consumed a token in a list List consumedTokenFutures; vint processBegin = previousBegin; vint processEnd = previousEnd; while (processEnd > processBegin) { // explore the current token if (state.ExploreStep(futures, processBegin, processEnd - processBegin, futures)) { CopyFrom( consumedTokenFutures, From(futures) .Skip(processEnd) .Take(futures.Count() - processEnd), true ); futures.RemoveRange(processEnd, futures.Count() - processEnd); } // explore left recursive reduce and normal reduce state.ExploreLeftRecursiveReduce(futures, processBegin, processEnd - processBegin, futures); state.ExploreNormalReduce(futures, processBegin, processEnd - processBegin, futures); // if a token is consumed, then for those reduce futures, explore them until a token is consumed, and discard all failed futures processBegin = processEnd; processEnd = futures.Count(); } if (consumedTokenFutures.Count() == 0) { // failed to get any future that consumed a token, do error recovering vint tokenIndex = state.GetCurrentTableTokenIndex(); OnErrorRecover(state, tokenIndex, futures, previousBegin, previousEnd, errors); if (previousBegin == previousEnd) { break; } } else { state.SkipCurrentToken(); // put all futures that consumed a token from consumedTokenFutures back to future list previousBegin = futures.Count(); CopyFrom(futures, consumedTokenFutures, true); previousEnd = futures.Count(); // resolve all futures and see if all futures collapsed into a equivalent single future resolvableFutureLevels = GetResolvableFutureLevels(futures, previousBegin, previousEnd); if (resolvableFutureLevels != 0) { break; } } } begin = previousBegin; end = previousEnd; return resolvableFutureLevels; } vint ParsingAmbiguousParser::GetConflictReduceCount(collections::List& futures) { vint conflictReduceCount=-1; for(vint i=0;iselectedItem->instructions.Count(); vint secondIndex=second->selectedItem->instructions.Count(); while(--firstIndex>=0 && --secondIndex>=0) { ParsingTable::Instruction* firstIns=&first->selectedItem->instructions[firstIndex]; ParsingTable::Instruction* secondIns=&second->selectedItem->instructions[secondIndex]; if(firstIns && secondIns) { if(firstIns->instructionType==secondIns->instructionType && firstIns->nameParameter==secondIns->nameParameter && firstIns->stateParameter==secondIns->stateParameter && firstIns->value==secondIns->value ) { if(firstIns->instructionType==ParsingTable::Instruction::Reduce || firstIns->instructionType==ParsingTable::Instruction::LeftRecursiveReduce) { count++; } continue; } } break; } if(conflictReduceCount==-1 || conflictReduceCount>count) { conflictReduceCount=count; } } if(conflictReduceCount==-1) { conflictReduceCount=0; } return conflictReduceCount; } void ParsingAmbiguousParser::GetConflictReduceIndices(collections::List& futures, vint conflictReduceCount, collections::Array& conflictReduceIndices) { conflictReduceIndices.Resize(futures.Count()); for(vint i=0;iselectedItem->instructions.Count(); vint count=0; while(count0) { switch(future->selectedItem->instructions[--index].instructionType) { case ParsingTable::Instruction::Reduce: case ParsingTable::Instruction::LeftRecursiveReduce: count++; break; default:; } } conflictReduceIndices[i]=index; } } vint ParsingAmbiguousParser::GetAffectedStackNodeCount(collections::List& futures, collections::Array& conflictReduceIndices) { vint affectedStackNodeCount=-1; for(vint i=0;iselectedItem) { vint start=(future==futures[i]?conflictReduceIndices[i]:future->selectedItem->instructions.Count())-1; for(vint j=start;j>=0;j--) { switch(future->selectedItem->instructions[j].instructionType) { case ParsingTable::Instruction::Reduce: count++; break; case ParsingTable::Instruction::Shift: count--; break; default:; } } future=future->previous; } if(affectedStackNodeCount==-1) { affectedStackNodeCount=count; } else if(affectedStackNodeCount!=count) { return -1; } } return affectedStackNodeCount; } void ParsingAmbiguousParser::BuildSingleDecisionPath(ParsingState& state, ParsingState::Future* future, vint lastAvailableInstructionCount) { List> path; while(future && future->selectedToken!=-1) { path.Add(Pair(future->selectedItem, future->selectedRegexToken)); future = future->previous; } for (vint j = path.Count() - 1; j >= 0; j--) { if(j==0 && lastAvailableInstructionCount!=-1) { decisions.Add(state.RunTransition(path[j].key, path[j].value, 0, lastAvailableInstructionCount, false)); } else { decisions.Add(state.RunTransition(path[j].key, path[j].value)); } } } void ParsingAmbiguousParser::BuildAmbiguousDecisions(ParsingState& state, collections::List& futures, vint begin, vint end, vint resolvableFutureLevels, collections::List>& errors) { List resolvingFutures; CopyFrom( resolvingFutures, From(futures) .Skip(begin) .Take(end - begin) ); for (vint i = 1; i < resolvableFutureLevels; i++) { for(vint j=0;jprevious; } } Array conflictReduceIndices; vint conflictReduceCount=GetConflictReduceCount(resolvingFutures); GetConflictReduceIndices(resolvingFutures, conflictReduceCount, conflictReduceIndices); WString ambiguityNodeType, ambiguityRuleName; if(resolvingFutures[0]->selectedItem->instructions.Count()==conflictReduceIndices[0]) { vint rootStartState=state.GetParsingRuleStartState(); ambiguityNodeType=state.GetTable()->GetStateInfo(rootStartState).ruleAmbiguousType; ambiguityRuleName=state.GetParsingRule(); } else { ParsingTable::Instruction& ins=resolvingFutures[0]->selectedItem->instructions[conflictReduceIndices[0]]; ambiguityNodeType=state.GetTable()->GetStateInfo(ins.stateParameter).ruleAmbiguousType; ambiguityRuleName=state.GetTable()->GetStateInfo(ins.stateParameter).ruleName; } if(ambiguityNodeType==L"") { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, L"Ambiguity happens when reducing rule \""+ambiguityRuleName+L"\" but this rule does not have an associated ambiguous node type."))); return; } vint affectedStackNodeCount=GetAffectedStackNodeCount(resolvingFutures, conflictReduceIndices); if(affectedStackNodeCount==-1) { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, (token==0?L"Failed to pass ambiguity checking during parsing when reaching to the end of the input.":L"Failed to pass ambiguity checking during parsing.")))); return; } Ptr stateGroup; for(vint i=0;iinstructions.Count()-start, true)); } } } ParsingState::Future* lastFuture=futures[end-1]; ParsingState::Future** futureCleaner=&lastFuture; for(int i=1;iprevious; } *futureCleaner=0; if(lastFuture) { BuildSingleDecisionPath(state, lastFuture, -1); } } void ParsingAmbiguousParser::BuildDecisions(ParsingState& state, collections::List& futures, vint begin, vint end, vint resolvableFutureLevels, collections::List>& errors) { if(end-begin==0) { const RegexToken* token=state.GetToken(state.GetCurrentToken()); errors.Add(Ptr(new ParsingError(token, (token==0?L"Error happened during parsing when reaching to the end of the input.":L"Error happened during parsing.")))); } else if(end-begin==1) { BuildSingleDecisionPath(state, futures[begin], -1); } else { BuildAmbiguousDecisions(state, futures, begin, end, resolvableFutureLevels, errors); } } ParsingState::TransitionResult ParsingAmbiguousParser::ParseStep(ParsingState& state, collections::List>& errors) { if(decisions.Count()==consumedDecisionCount) { List futures; vint resultBegin=0; vint resultEnd=0; vint resolvableFutureLevels=SearchPathForOneStep(state, futures, resultBegin, resultEnd, errors); BuildDecisions(state, futures, resultBegin, resultEnd, resolvableFutureLevels, errors); for (auto future : futures) { delete future; } } if(decisions.Count()>consumedDecisionCount) { ParsingState::TransitionResult result=decisions[consumedDecisionCount++]; if(consumedDecisionCount==decisions.Count()) { decisions.Clear(); consumedDecisionCount=0; } return result; } else { return ParsingState::TransitionResult(); } } void ParsingAmbiguousParser::BeginParse() { decisions.Clear(); consumedDecisionCount=0; ParsingGeneralParser::BeginParse(); } /*********************************************************************** ParsingAutoRecoverAmbiguousParser ***********************************************************************/ void ParsingAutoRecoverAmbiguousParser::OnErrorRecover(ParsingState& state, vint currentTokenIndex, collections::List& futures, vint& begin, vint& end, collections::List>& errors) { vint insertedTokenCount = 0; while (insertedTokenCount++ < maxInsertedTokenCount) { // keep all futures that consumed a token in a list List consumedTokenFutures; vint processBegin = begin; vint processEnd = end; while (processEnd > processBegin) { // explore all tokens for (vint i = processBegin; i < processEnd; i++) { state.Explore(ParsingTable::TokenFinish, futures[i], futures); for (vint token = ParsingTable::UserTokenStart; token < state.GetTable()->GetTokenCount(); token++) { state.Explore(token, futures[i], futures); } } // copy all futures that consumed a token to consumedTokenFutures if (futures.Count() > processEnd) { CopyFrom( consumedTokenFutures, From(futures) .Skip(processEnd) .Take(futures.Count() - processEnd), true ); futures.RemoveRange(processEnd, futures.Count() - processEnd); } // explore left recursive reduce and normal reduce state.ExploreLeftRecursiveReduce(futures, processBegin, processEnd - processBegin, futures); state.ExploreNormalReduce(futures, processBegin, processEnd - processBegin, futures); // if a token is consumed, then for those reduce futures, explore them until a token is consumed, and discard all failed futures processBegin = processEnd; processEnd = futures.Count(); } if (consumedTokenFutures.Count() == 0) { // failed to get any future that consumed a token goto ERROR_RECOVERY_FAILED; } else { // try to see if the target token is reached List recoveryFutures; for (auto future : consumedTokenFutures) { if (future->selectedToken == currentTokenIndex) { // because this is reached by error recoverying, so all futures in availableFutures should have previous futures recoveryFutures.Add(future->previous); } } if (recoveryFutures.Count()>0) { // finally reached the expected currentTokenIndex // move these previous futures to the end // then the original parser algorith, will use these previous futures to reach the currentTokenIndex in the next step for (auto future : recoveryFutures) { futures.Remove(future); futures.Add(future); } begin = futures.Count() - recoveryFutures.Count(); end = futures.Count(); // delete all futures in consumedTokenFutures for (auto future : consumedTokenFutures) { delete future; } goto ERROR_RECOVERY_SUCCEEDED; } else { // put all futures that consumed a token from consumedTokenFutures back to future list begin = futures.Count(); CopyFrom(futures, consumedTokenFutures, true); end = futures.Count(); } } } // if the maxInsertedTokenCount is exceeded, then we get here ERROR_RECOVERY_FAILED: begin = end = futures.Count(); return; ERROR_RECOVERY_SUCCEEDED: return; } ParsingAutoRecoverAmbiguousParser::ParsingAutoRecoverAmbiguousParser(Ptr _table, vint _maxInsertedTokenCount) :ParsingAmbiguousParser(_table) , maxInsertedTokenCount(_maxInsertedTokenCount == -1 ? 4 : _maxInsertedTokenCount) { } ParsingAutoRecoverAmbiguousParser::~ParsingAutoRecoverAmbiguousParser() { } /*********************************************************************** Helper Functions ***********************************************************************/ Ptr CreateStrictParser(Ptr table) { if(table) { if(table->GetAmbiguity()) { return Ptr(new ParsingAmbiguousParser(table)); } else { return Ptr(new ParsingStrictParser(table)); } } else { return 0; } } Ptr CreateAutoRecoverParser(Ptr table) { if(table) { if(table->GetAmbiguity()) { return Ptr(new ParsingAutoRecoverAmbiguousParser(table)); } else { return Ptr(new ParsingAutoRecoverParser(table)); } } else { return 0; } } Ptr CreateBootstrapStrictParser() { List> errors; Ptr definition=CreateParserDefinition(); Ptr table=GenerateTable(definition, false, errors); return CreateStrictParser(table); } Ptr CreateBootstrapAutoRecoverParser() { List> errors; Ptr definition=CreateParserDefinition(); Ptr table=GenerateTable(definition, false, errors); return CreateAutoRecoverParser(table); } } } } /*********************************************************************** Reflection ***********************************************************************/ #ifndef VCZH_DEBUG_NO_REFLECTION namespace vl { namespace reflection { namespace description { using namespace parsing; PARSINGREFLECTION_TYPELIST(IMPL_VL_TYPE_INFO) /*********************************************************************** Type Declaration ***********************************************************************/ #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA #define _ , BEGIN_STRUCT_MEMBER(ParsingTextPos) STRUCT_MEMBER(index) STRUCT_MEMBER(row) STRUCT_MEMBER(column) END_STRUCT_MEMBER(ParsingTextPos) BEGIN_STRUCT_MEMBER(ParsingTextRange) STRUCT_MEMBER(start) STRUCT_MEMBER(end) END_STRUCT_MEMBER(ParsingTextRange) BEGIN_CLASS_MEMBER(ParsingTreeNode) CLASS_MEMBER_PROPERTY_FAST(CodeRange) CLASS_MEMBER_PROPERTY_READONLY_FAST(Parent) CLASS_MEMBER_PROPERTY_READONLY_FAST(SubNodes) CLASS_MEMBER_METHOD(Clone, NO_PARAMETER) CLASS_MEMBER_METHOD(InitializeQueryCache, NO_PARAMETER) CLASS_MEMBER_METHOD(ClearQueryCache, NO_PARAMETER) CLASS_MEMBER_METHOD_OVERLOAD(FindSubNode, {L"position"}, ParsingTreeNode*(ParsingTreeNode::*)(const ParsingTextPos&)) CLASS_MEMBER_METHOD_OVERLOAD(FindSubNode, {L"range"}, ParsingTreeNode*(ParsingTreeNode::*)(const ParsingTextRange&)) CLASS_MEMBER_METHOD_OVERLOAD(FindDeepestNode, {L"position"}, ParsingTreeNode*(ParsingTreeNode::*)(const ParsingTextPos&)) CLASS_MEMBER_METHOD_OVERLOAD(FindDeepestNode, {L"range"}, ParsingTreeNode*(ParsingTreeNode::*)(const ParsingTextRange&)) END_CLASS_MEMBER(ParsingTreeNode) BEGIN_CLASS_MEMBER(ParsingTreeToken) CLASS_MEMBER_CONSTRUCTOR(Ptr(const WString&, vint), {L"value" _ L"tokenIndex"}) CLASS_MEMBER_PROPERTY_FAST(TokenIndex) CLASS_MEMBER_PROPERTY_FAST(Value) END_CLASS_MEMBER(ParsingTreeToken) BEGIN_CLASS_MEMBER(ParsingTreeObject) CLASS_MEMBER_CONSTRUCTOR(Ptr(const WString&), {L"type"}) CLASS_MEMBER_PROPERTY_FAST(Type) CLASS_MEMBER_PROPERTY_READONLY_FAST(Members) CLASS_MEMBER_PROPERTY_READONLY_FAST(MemberNames) CLASS_MEMBER_PROPERTY_READONLY_FAST(CreatorRules) CLASS_MEMBER_METHOD(GetMember, {L"name"}) CLASS_MEMBER_METHOD(SetMember, {L"name" _ L"node"}) END_CLASS_MEMBER(ParsingTreeObject) BEGIN_CLASS_MEMBER(ParsingTreeArray) CLASS_MEMBER_CONSTRUCTOR(Ptr(const WString&), {L"elementType"}) CLASS_MEMBER_PROPERTY_FAST(ElementType) CLASS_MEMBER_PROPERTY_READONLY_FAST(Items) CLASS_MEMBER_METHOD(GetItem, {L"index"}) CLASS_MEMBER_METHOD(SetItem, {L"index" _ L"node"}) CLASS_MEMBER_METHOD(AddItem, {L"node"}) CLASS_MEMBER_METHOD(InsertItem, {L"index" _ L"node"}) CLASS_MEMBER_METHOD_OVERLOAD(RemoveItem, {L"index"}, bool(ParsingTreeArray::*)(vint)) CLASS_MEMBER_METHOD_OVERLOAD(RemoveItem, {L"node"}, bool(ParsingTreeArray::*)(ParsingTreeNode*)) CLASS_MEMBER_METHOD(IndexOfItem, {L"node"}) CLASS_MEMBER_METHOD(ContainsItem, {L"node"}) CLASS_MEMBER_METHOD(Clone, NO_PARAMETER) CLASS_MEMBER_METHOD_RENAME(GetCount, Count, NO_PARAMETER) CLASS_MEMBER_PROPERTY_READONLY(Count, GetCount) END_CLASS_MEMBER(ParsingTreeArray) BEGIN_CLASS_MEMBER(ParsingTreeCustomBase) CLASS_MEMBER_FIELD(codeRange) CLASS_MEMBER_FIELD(creatorRules) END_CLASS_MEMBER(ParsingTreeCustomBase) BEGIN_CLASS_MEMBER(ParsingToken) CLASS_MEMBER_BASE(ParsingTreeCustomBase) CLASS_MEMBER_FIELD(tokenIndex) CLASS_MEMBER_FIELD(value) END_CLASS_MEMBER(ParsingToken) BEGIN_CLASS_MEMBER(ParsingError) CLASS_MEMBER_CONSTRUCTOR(Ptr(), NO_PARAMETER) CLASS_MEMBER_CONSTRUCTOR(Ptr(const WString&), {L"errorMessage"}) CLASS_MEMBER_CONSTRUCTOR(Ptr(ParsingTreeCustomBase*, const WString&), {L"parsingTree" _ L"errorMessage"}) CLASS_MEMBER_FIELD(codeRange) CLASS_MEMBER_FIELD(parsingTree) CLASS_MEMBER_FIELD(errorMessage) END_CLASS_MEMBER(ParsingError) #undef _ #endif } } } #endif namespace vl { namespace reflection { namespace description { /*********************************************************************** Type Loader ***********************************************************************/ #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA class ParsingTypeLoader : public Object, public ITypeLoader { public: void Load(ITypeManager* manager) { PARSINGREFLECTION_TYPELIST(ADD_TYPE_INFO) } void Unload(ITypeManager* manager) { } }; #endif bool LoadParsingTypes() { #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA ITypeManager* manager=GetGlobalTypeManager(); if(manager) { auto loader = Ptr(new ParsingTypeLoader); return manager->AddTypeLoader(loader); } #endif return false; } } } } /*********************************************************************** .\PARSINGANALYZER.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; namespace analyzing { /*********************************************************************** ParsingSymbol ***********************************************************************/ bool ParsingSymbol::AddSubSymbol(ParsingSymbol* subSymbol) { if(subSymbol->GetParentSymbol()) return false; if(subSymbolMap.Keys().IndexOf(subSymbol->GetName())!=-1) return false; switch(type) { case Global: switch(subSymbol->GetType()) { case EnumType: break; case ClassType: break; case TokenDef: break; case RuleDef: break; default: return false; } break; case EnumType: switch(subSymbol->GetType()) { case EnumItem: break; default: return false; } break; case ClassType: switch(subSymbol->GetType()) { case EnumType: case ClassType: case ClassField: break; default: return false; } break; default: return false; } subSymbol->parentSymbol=this; subSymbolList.Add(subSymbol); subSymbolMap.Add(subSymbol->GetName(), subSymbol); return true; } ParsingSymbol::ParsingSymbol(ParsingSymbolManager* _manager, SymbolType _type, const WString& _name, ParsingSymbol* _descriptorSymbol, const WString& _descriptorString) :manager(_manager) ,type(_type) ,name(_name) ,descriptorSymbol(_descriptorSymbol) ,descriptorString(_descriptorString) ,parentSymbol(0) ,arrayTypeSymbol(0) { } ParsingSymbol::~ParsingSymbol() { } ParsingSymbolManager* ParsingSymbol::GetManager() { return manager; } ParsingSymbol::SymbolType ParsingSymbol::GetType() { return type; } const WString& ParsingSymbol::GetName() { return name; } vint ParsingSymbol::GetSubSymbolCount() { return subSymbolList.Count(); } ParsingSymbol* ParsingSymbol::GetSubSymbol(vint index) { if(0<=index && indexGetSubSymbolByName(name); if(subSymbol) { return subSymbol; } else { scope=scope->GetDescriptorSymbol(); } } } return 0; } ParsingSymbol* ParsingSymbol::SearchCommonBaseClass(ParsingSymbol* classType) { if(type==ParsingSymbol::ClassType && classType->GetType()==ParsingSymbol::ClassType) { vint aCount=0; vint bCount=0; ParsingSymbol* a=this; ParsingSymbol* b=classType; while(a || b) { if(a) { aCount++; a=a->GetDescriptorSymbol(); } if(b) { bCount++; b=b->GetDescriptorSymbol(); } } a=this; b=classType; vint min=aCountmin;i--) { a=a->GetDescriptorSymbol(); } for(vint i=bCount;i>min;i--) { b=b->GetDescriptorSymbol(); } while(a!=b) { a=a->GetDescriptorSymbol(); b=b->GetDescriptorSymbol(); } return a; } return 0; } /*********************************************************************** ParsingSymbolManager ***********************************************************************/ bool ParsingSymbolManager::TryAddSubSymbol(Ptr subSymbol, ParsingSymbol* parentSymbol) { if(parentSymbol->AddSubSymbol(subSymbol.Obj())) { createdSymbols.Add(subSymbol); return true; } return false; } ParsingSymbolManager::ParsingSymbolManager() { globalSymbol=new ParsingSymbol(this, ParsingSymbol::Global, L"", 0, L""); tokenTypeSymbol=new ParsingSymbol(this, ParsingSymbol::TokenType, L"token", 0, L""); createdSymbols.Add(Ptr(globalSymbol)); createdSymbols.Add(Ptr(tokenTypeSymbol)); } ParsingSymbolManager::~ParsingSymbolManager() { } ParsingSymbol* ParsingSymbolManager::GetGlobal() { return globalSymbol; } ParsingSymbol* ParsingSymbolManager::GetTokenType() { return tokenTypeSymbol; } ParsingSymbol* ParsingSymbolManager::GetArrayType(ParsingSymbol* elementType) { if(elementType->IsType()) { if(!elementType->arrayTypeSymbol) { elementType->arrayTypeSymbol=new ParsingSymbol(this, ParsingSymbol::ArrayType, L"", elementType, L""); createdSymbols.Add(Ptr(elementType->arrayTypeSymbol)); } return elementType->arrayTypeSymbol; } else { return nullptr; } } ParsingSymbol* ParsingSymbolManager::AddClass(definitions::ParsingDefinitionClassDefinition* classDef, ParsingSymbol* baseType, ParsingSymbol* parentType) { if((!baseType || baseType->GetType()==ParsingSymbol::ClassType) && (!parentType || parentType->IsType())) { auto symbol = Ptr(new ParsingSymbol(this, ParsingSymbol::ClassType, classDef->name, baseType, L"")); if(TryAddSubSymbol(symbol, parentType?parentType:globalSymbol)) { symbolClassDefinitionCache.Add(symbol.Obj(), classDef); classDefinitionSymbolCache.Add(classDef, symbol.Obj()); return symbol.Obj(); } } return nullptr; } ParsingSymbol* ParsingSymbolManager::AddField(const WString& name, ParsingSymbol* classType, ParsingSymbol* fieldType) { if(classType && classType->GetType()==ParsingSymbol::ClassType && fieldType && fieldType->IsType()) { auto symbol = Ptr(new ParsingSymbol(this, ParsingSymbol::ClassField, name, fieldType, L"")); if(TryAddSubSymbol(symbol, classType)) { return symbol.Obj(); } } return nullptr; } ParsingSymbol* ParsingSymbolManager::AddEnum(const WString& name, ParsingSymbol* parentType) { if(!parentType || parentType->GetType()==ParsingSymbol::ClassType) { auto symbol = Ptr(new ParsingSymbol(this, ParsingSymbol::EnumType, name, 0, L"")); if(TryAddSubSymbol(symbol, parentType?parentType:globalSymbol)) { return symbol.Obj(); } } return nullptr; } ParsingSymbol* ParsingSymbolManager::AddEnumItem(const WString& name, ParsingSymbol* enumType) { if(enumType && enumType->GetType()==ParsingSymbol::EnumType) { auto symbol = Ptr(new ParsingSymbol(this, ParsingSymbol::EnumItem, name, enumType, L"")); if(TryAddSubSymbol(symbol, enumType)) { return symbol.Obj(); } } return nullptr; } ParsingSymbol* ParsingSymbolManager::AddTokenDefinition(const WString& name, const WString& regex) { auto symbol = Ptr(new ParsingSymbol(this, ParsingSymbol::TokenDef, name, tokenTypeSymbol, regex)); if(TryAddSubSymbol(symbol, globalSymbol)) { return symbol.Obj(); } return nullptr; } ParsingSymbol* ParsingSymbolManager::AddRuleDefinition(const WString& name, ParsingSymbol* ruleType) { if(ruleType && ruleType->IsType()) { auto symbol = Ptr(new ParsingSymbol(this, ParsingSymbol::RuleDef, name, ruleType, L"")); if(TryAddSubSymbol(symbol, globalSymbol)) { return symbol.Obj(); } } return nullptr; } ParsingSymbolManager::ClassDefinition* ParsingSymbolManager::CacheGetClassDefinition(ParsingSymbol* type) { vint index=symbolClassDefinitionCache.Keys().IndexOf(type); return index==-1?0:symbolClassDefinitionCache.Values().Get(index); } ParsingSymbol* ParsingSymbolManager::CacheGetClassType(ClassDefinition* type) { vint index=classDefinitionSymbolCache.Keys().IndexOf(type); return index==-1?0:classDefinitionSymbolCache.Values().Get(index); } ParsingSymbol* ParsingSymbolManager::CacheGetType(definitions::ParsingDefinitionType* type, ParsingSymbol* scope) { DefinitionTypeScopePair key(type, scope); vint index=definitionTypeSymbolCache.Keys().IndexOf(key); return index==-1?0:definitionTypeSymbolCache.Values().Get(index); } bool ParsingSymbolManager::CacheSetType(definitions::ParsingDefinitionType* type, ParsingSymbol* scope, ParsingSymbol* symbol) { DefinitionTypeScopePair key(type, scope); vint index=definitionTypeSymbolCache.Keys().IndexOf(key); if(index==-1) { definitionTypeSymbolCache.Add(key, symbol); return true; } else { return false; } } ParsingSymbol* ParsingSymbolManager::CacheGetSymbol(definitions::ParsingDefinitionGrammar* grammar) { vint index=definitionGrammarSymbolCache.Keys().IndexOf(grammar); return index==-1?0:definitionGrammarSymbolCache.Values().Get(index); } bool ParsingSymbolManager::CacheSetSymbol(definitions::ParsingDefinitionGrammar* grammar, ParsingSymbol* symbol) { vint index=definitionGrammarSymbolCache.Keys().IndexOf(grammar); if(index==-1) { definitionGrammarSymbolCache.Add(grammar, symbol); return true; } else { return false; } } ParsingSymbol* ParsingSymbolManager::CacheGetType(definitions::ParsingDefinitionGrammar* grammar) { vint index=definitionGrammarTypeCache.Keys().IndexOf(grammar); return index==-1?0:definitionGrammarTypeCache.Values().Get(index); } bool ParsingSymbolManager::CacheSetType(definitions::ParsingDefinitionGrammar* grammar, ParsingSymbol* type) { vint index=definitionGrammarTypeCache.Keys().IndexOf(grammar); if(index==-1) { definitionGrammarTypeCache.Add(grammar, type); return true; } else { return false; } } /*********************************************************************** FindType ***********************************************************************/ WString GetTypeFullName(ParsingSymbol* type) { if(type->GetType()==ParsingSymbol::ArrayType) { return GetTypeFullName(type->GetDescriptorSymbol())+L"[]"; } else { WString name=type->GetName(); type=type->GetParentSymbol(); while(type && type!=type->GetManager()->GetGlobal()) { name=type->GetName()+L"."+name; type=type->GetParentSymbol(); } return name; } } /*********************************************************************** FindType ***********************************************************************/ class FindTypeVisitor : public Object, public ParsingDefinitionType::IVisitor { public: ParsingSymbolManager* manager; ParsingSymbol* scope; List>& errors; ParsingSymbol* result; FindTypeVisitor(ParsingSymbolManager* _manager, ParsingSymbol* _scope, List>& _errors) :manager(_manager) ,scope(_scope) ,errors(_errors) ,result(0) { } void Visit(ParsingDefinitionPrimitiveType* node)override { ParsingSymbol* currentScope=scope; while(currentScope) { ParsingSymbol* type=currentScope->GetSubSymbolByName(node->name); if(type) { if(type->IsType()) { result=type; } else { errors.Add(Ptr(new ParsingError(node, L"\"" + node->name + L"\" in current scope is not a type."))); } return; } currentScope=currentScope->GetParentSymbol(); } errors.Add(Ptr(new ParsingError(node, L"Cannot not find \"" + node->name + L"\" in current scope."))); } void Visit(ParsingDefinitionTokenType* node)override { result=manager->GetTokenType(); } void Visit(ParsingDefinitionSubType* node)override { ParsingSymbol* type=FindType(node->parentType.Obj(), manager, scope, errors); if(type) { ParsingSymbol* subType=type->SearchClassSubSymbol(node->subTypeName); if(!subType) { errors.Add(Ptr(new ParsingError(node, L"\"" + GetTypeFullName(type) + L"\" does not has a sub type called \"" + node->subTypeName + L"\"."))); } else if(subType->IsType()) { result=subType; } else { errors.Add(Ptr(new ParsingError(node, L"\"" + GetTypeFullName(type) + L"\" contains a sub definition called \"" + node->subTypeName + L"\" but this is not a type."))); } } } void Visit(ParsingDefinitionArrayType* node)override { ParsingSymbol* type=FindType(node->elementType.Obj(), manager, scope, errors); if(type) { result=manager->GetArrayType(type); } } }; ParsingSymbol* FindType(definitions::ParsingDefinitionType* type, ParsingSymbolManager* manager, ParsingSymbol* scope, collections::List>& errors) { ParsingSymbol* result=manager->CacheGetType(type, scope); if(!result) { FindTypeVisitor visitor(manager, (scope?scope:manager->GetGlobal()), errors); type->Accept(&visitor); result=visitor.result; manager->CacheSetType(type, scope, result); } return result; } /*********************************************************************** PrepareSymbols ***********************************************************************/ class PrepareSymbolsTypeDefinitionVisitor : public Object, public ParsingDefinitionTypeDefinition::IVisitor { public: ParsingSymbolManager* manager; ParsingSymbol* scope; List>& errors; PrepareSymbolsTypeDefinitionVisitor(ParsingSymbolManager* _manager, ParsingSymbol* _scope, List>& _errors) :manager(_manager) ,scope(_scope) ,errors(_errors) { } bool EnsureNameNotExists(ParsingDefinitionTypeDefinition* node, const WString& subjectName) { if(scope->SearchClassSubSymbol(node->name)) { errors.Add(Ptr(new ParsingError(node, L"Cannot redefine \"" + node->name + L"\" to be " + subjectName + L"."))); return false; } else { return true; } } void Visit(ParsingDefinitionClassMemberDefinition* node)override { if(EnsureNameNotExists(node, L"a class field")) { ParsingSymbol* fieldType=FindType(node->type.Obj(), manager, scope, errors); if(fieldType) { ParsingSymbol* field=manager->AddField(node->name, scope, fieldType); if(!field) { errors.Add(Ptr(new ParsingError(node, L"A class field cannot be defined here."))); } } } } void Visit(ParsingDefinitionClassDefinition* node)override { if(EnsureNameNotExists(node, L"a class type")) { ParsingSymbol* baseType=0; if(node->parentType) { baseType=FindType(node->parentType.Obj(), manager, scope, errors); } ParsingSymbol* classType=manager->AddClass(node, baseType, (scope->GetType()==ParsingSymbol::Global?0:scope)); if(classType) { PrepareSymbolsTypeDefinitionVisitor visitor(manager, classType, errors); for (auto subType : node->subTypes) { subType->Accept(&visitor); } for (auto member : node->members) { member->Accept(&visitor); } } else { errors.Add(Ptr(new ParsingError(node, L"A class type cannot be defined here."))); } } } void Visit(ParsingDefinitionEnumMemberDefinition* node)override { if(EnsureNameNotExists(node, L"an enum item")) { ParsingSymbol* enumItem=manager->AddEnumItem(node->name, scope); if(!enumItem) { errors.Add(Ptr(new ParsingError(node, L"An enum item cannot be defined here."))); } } } void Visit(ParsingDefinitionEnumDefinition* node)override { if(EnsureNameNotExists(node, L"an enum type")) { ParsingSymbol* enumType=manager->AddEnum(node->name, (scope->GetType()==ParsingSymbol::Global?0:scope)); if(enumType) { PrepareSymbolsTypeDefinitionVisitor visitor(manager, enumType, errors); for (auto member : node->members) { member->Accept(&visitor); } } else { errors.Add(Ptr(new ParsingError(node, L"An enum type cannot be defined here."))); } } } }; void PrepareSymbols(Ptr definition, ParsingSymbolManager* manager, collections::List>& errors) { { PrepareSymbolsTypeDefinitionVisitor visitor(manager, manager->GetGlobal(), errors); for (auto typeDefinition : definition->types) { typeDefinition->Accept(&visitor); } } for (auto token : definition->tokens) { if(manager->GetGlobal()->GetSubSymbolByName(token->name)) { errors.Add(Ptr(new ParsingError(token.Obj(), L"Cannot redefine \""+token->name+L"\" to be a token definition."))); } else { manager->AddTokenDefinition(token->name, token->regex); try { regex_internal::ParseRegexExpression(wtou32(token->regex)); } catch(const regex_internal::RegexException& ex) { errors.Add(Ptr(new ParsingError(token.Obj(), L"Wrong token definition for \""+token->name+L"\": "+ex.Message()))); } } } for (auto rule : definition->rules) { if(manager->GetGlobal()->GetSubSymbolByName(rule->name)) { errors.Add(Ptr(new ParsingError(rule.Obj(), L"Cannot redefine \""+rule->name+L"\" to be a rule definition."))); } else { ParsingSymbol* type=FindType(rule->type.Obj(), manager, 0, errors); if(type) { if(type->GetType()!=ParsingSymbol::ClassType) { errors.Add(Ptr(new ParsingError(rule.Obj(), L"\""+GetTypeFullName(type)+L"\" cannot be a type of a rule because this is not a class type."))); } manager->AddRuleDefinition(rule->name, type); } } } } /*********************************************************************** ValidateRuleStructure ***********************************************************************/ class ValidateRuleStructureVisitor : public Object, public ParsingDefinitionGrammar::IVisitor { public: Ptr definition; ParsingSymbolManager* manager; ParsingDefinitionRuleDefinition* rule; List>& errors; vint loopCount; ValidateRuleStructureVisitor(Ptr _definition, ParsingSymbolManager* _manager, ParsingDefinitionRuleDefinition* _rule, List>& _errors) :definition(_definition) ,manager(_manager) ,errors(_errors) ,rule(_rule) ,loopCount(0) { } void CheckCreationType(ParsingDefinitionGrammar* node, ParsingSymbol* nodeType) { if(nodeType->GetType()==ParsingSymbol::ClassType) { ParsingSymbol* ruleType=manager->GetGlobal()->GetSubSymbolByName(rule->name)->GetDescriptorSymbol(); ParsingSymbol* currentType=nodeType; while(currentType && currentType!=ruleType) { currentType=currentType->GetDescriptorSymbol(); } if(!currentType) { errors.Add(Ptr(new ParsingError(node, L"Cannot create type \""+GetTypeFullName(nodeType)+L"\" in a rule of type \""+GetTypeFullName(ruleType)+L"\" because there are no implicit conversions from the created type to the rule type."))); } } else { errors.Add(Ptr(new ParsingError(node, L"\""+GetTypeFullName(nodeType)+L"\" cannot be created because this is not a class type."))); } } void Visit(ParsingDefinitionPrimitiveGrammar* node)override { ParsingSymbol* symbol=manager->GetGlobal()->GetSubSymbolByName(node->name); if(!symbol) { errors.Add(Ptr(new ParsingError(node, L"Cannot find a token or a rule with name \""+node->name+L"\"."))); } else switch(symbol->GetType()) { case ParsingSymbol::TokenDef: { bool discard=false; for (auto token : definition->tokens) { if(token->name==symbol->GetName()) { discard=token->discard; break; } } if(discard) { errors.Add(Ptr(new ParsingError(node, L"Cannot use discard token \""+node->name+L"\" as input."))); break; } } case ParsingSymbol::RuleDef: { ParsingSymbol* symbolType=symbol->GetDescriptorSymbol(); manager->CacheSetSymbol(node, symbol); manager->CacheSetType(node, symbolType); } break; default: errors.Add(Ptr(new ParsingError(node, L"\""+node->name+L"\" is not a token definition or rule definition."))); } } void Visit(ParsingDefinitionTextGrammar* node)override { auto regex = regex_internal::EscapeTextForRegex(wtou32(node->text)); for (vint i = 0; i < manager->GetGlobal()->GetSubSymbolCount(); i++) { ParsingSymbol* symbol = manager->GetGlobal()->GetSubSymbol(i); if (symbol->GetType() == ParsingSymbol::TokenDef) { auto normalizedRegex = regex_internal::NormalizeEscapedTextForRegex(wtou32(symbol->GetDescriptorString())); if (normalizedRegex == regex) { manager->CacheSetSymbol(node, symbol); manager->CacheSetType(node, manager->GetTokenType()); return; } } } errors.Add(Ptr(new ParsingError(node, L"Cannot find a token whose definition is exactly \"" + u32tow(regex) + L"\"."))); } void Visit(ParsingDefinitionSequenceGrammar* node)override { node->first->Accept(this); node->second->Accept(this); } void Visit(ParsingDefinitionAlternativeGrammar* node)override { node->first->Accept(this); node->second->Accept(this); } void Visit(ParsingDefinitionLoopGrammar* node)override { loopCount++; node->grammar->Accept(this); loopCount--; } void Visit(ParsingDefinitionOptionalGrammar* node)override { node->grammar->Accept(this); } void Visit(ParsingDefinitionCreateGrammar* node)override { if(loopCount>0) { errors.Add(Ptr(new ParsingError(node, L"Parsing tree node creation (the \"as\" operator) is not allowed inside loops."))); } if(ParsingSymbol* nodeType=FindType(node->type.Obj(), manager, 0, errors)) { CheckCreationType(node, nodeType); } node->grammar->Accept(this); } void Visit(ParsingDefinitionAssignGrammar* node)override { if(!node->grammar.Cast() && !node->grammar.Cast()) { errors.Add(Ptr(new ParsingError(node, L"Only parsing tree node returned from a rule or a token can be assigned to a class field."))); } node->grammar->Accept(this); } void Visit(ParsingDefinitionUseGrammar* node)override { if(loopCount>0) { errors.Add(Ptr(new ParsingError(node, L"Parsing tree node reusing (the \"!\" operator) is not allowed inside loops."))); } if(!node->grammar.Cast()) { errors.Add(Ptr(new ParsingError(node, L"Only parsing tree node returned from a rule can be reused."))); } else if(ParsingSymbol* symbol=manager->CacheGetSymbol(node->grammar.Obj())) { if(symbol->GetType()!=ParsingSymbol::RuleDef) { errors.Add(Ptr(new ParsingError(node, L"Only parsing tree node returned from a rule can be reused."))); } } if(ParsingSymbol* nodeType=manager->CacheGetType(node->grammar.Obj())) { CheckCreationType(node, nodeType); } node->grammar->Accept(this); } void Visit(ParsingDefinitionSetterGrammar* node)override { node->grammar->Accept(this); } }; void ValidateRuleStructure(Ptr definition, Ptr rule, ParsingSymbolManager* manager, collections::List>& errors) { for (auto grammar : rule->grammars) { ValidateRuleStructureVisitor visitor(definition, manager, rule.Obj(), errors); grammar->Accept(&visitor); } } /*********************************************************************** ResolveRuleSymbols ***********************************************************************/ struct GrammarPathFragment { // primitive, text -> transition // loop, optional, create, use assign, setter -> epsilon GrammarPathFragment* previousFragment; ParsingDefinitionGrammar* grammar; bool epsilon; ParsingSymbol* createdType; GrammarPathFragment() :previousFragment(0) ,grammar(0) ,epsilon(false) ,createdType(0) { } }; struct GrammarPath { List> fragments; ParsingSymbol* pathType; GrammarPath() :pathType(0) { } WString ToString() { WString result; for (auto fragment : fragments) { if(!fragment->epsilon) { if(result!=L"") result+=L" "; result+=GrammarToString(fragment->grammar); } } return result; } }; struct GrammarPathContainer { List> paths; }; class EnumerateGrammarPathVisitor : public Object, public ParsingDefinitionGrammar::IVisitor { public: ParsingSymbolManager* manager; ParsingDefinitionRuleDefinition* rule; List> createdFragments; List currentFragmentEnds; EnumerateGrammarPathVisitor(ParsingSymbolManager* _manager, ParsingDefinitionRuleDefinition* _rule) :manager(_manager) ,rule(_rule) { } EnumerateGrammarPathVisitor(const EnumerateGrammarPathVisitor& visitor) :manager(visitor.manager) ,rule(visitor.rule) { CopyFrom(currentFragmentEnds, visitor.currentFragmentEnds); } void Join(const EnumerateGrammarPathVisitor& visitor) { CopyFrom(createdFragments, visitor.createdFragments, true); CopyFrom(currentFragmentEnds, visitor.currentFragmentEnds, true); } void AddFragment(ParsingDefinitionGrammar* node, bool epsilon, ParsingSymbol* createdType) { if(currentFragmentEnds.Count()==0) { auto fragment = Ptr(new GrammarPathFragment); fragment->grammar=node; fragment->epsilon=epsilon; fragment->createdType=createdType; createdFragments.Add(fragment); currentFragmentEnds.Add(fragment.Obj()); } else for(vint i=0;igrammar=node; fragment->epsilon=epsilon; fragment->createdType=createdType; createdFragments.Add(fragment); fragment->previousFragment=currentFragmentEnds[i]; currentFragmentEnds[i] = fragment.Obj(); } } void BuildPath(List>& paths) { for (auto fragment : currentFragmentEnds) { auto path = Ptr(new GrammarPath); paths.Add(path); GrammarPathFragment* current=fragment; while(current) { path->fragments.Insert(0, createdFragments[createdFragments.IndexOf(current)]); current=current->previousFragment; } } } void Visit(ParsingDefinitionPrimitiveGrammar* node)override { AddFragment(node, false, 0); } void Visit(ParsingDefinitionTextGrammar* node)override { AddFragment(node, false, 0); } void Visit(ParsingDefinitionSequenceGrammar* node)override { node->first->Accept(this); node->second->Accept(this); } void Visit(ParsingDefinitionAlternativeGrammar* node)override { EnumerateGrammarPathVisitor visitor(*this); node->second->Accept(&visitor); node->first->Accept(this); Join(visitor); } void Visit(ParsingDefinitionLoopGrammar* node)override { EnumerateGrammarPathVisitor visitor(*this); node->grammar->Accept(&visitor); AddFragment(node, true, 0); Join(visitor); } void Visit(ParsingDefinitionOptionalGrammar* node)override { EnumerateGrammarPathVisitor visitor(*this); node->grammar->Accept(&visitor); AddFragment(node, true, 0); Join(visitor); } void Visit(ParsingDefinitionCreateGrammar* node)override { node->grammar->Accept(this); AddFragment(node, true, manager->CacheGetType(node->type.Obj(), 0)); } void Visit(ParsingDefinitionAssignGrammar* node)override { node->grammar->Accept(this); AddFragment(node, true, 0); } void Visit(ParsingDefinitionUseGrammar* node)override { node->grammar->Accept(this); AddFragment(node, true, manager->CacheGetSymbol(node->grammar.Obj())->GetDescriptorSymbol()); } void Visit(ParsingDefinitionSetterGrammar* node)override { node->grammar->Accept(this); AddFragment(node, true, 0); } }; class ResolveAssignerGrammarVisitor : public Object, public ParsingDefinitionGrammar::IVisitor { public: typedef Dictionary> GrammarPathMap; ParsingSymbolManager* manager; List>& errors; GrammarPathMap& grammarPaths; ResolveAssignerGrammarVisitor(ParsingSymbolManager* _manager, List>& _errors, GrammarPathMap& _grammarPaths) :manager(_manager) ,errors(_errors) ,grammarPaths(_grammarPaths) { } ParsingSymbol* GetFieldFromCombined(ParsingDefinitionGrammar* node, const WString& fieldName) { Ptr paths=grammarPaths[node]; ParsingSymbol* pathType=paths->paths[0]->pathType; for(vint i=1;ipaths.Count();i++) { pathType=pathType->SearchCommonBaseClass(paths->paths[i]->pathType); if(!pathType) break; } WString pathNames; WString typeNames; for(int i=0;ipaths.Count();i++) { if(i>0) { pathNames+=L", "; typeNames+=L", "; } pathNames+=L"{"+paths->paths[i]->ToString()+L"}"; typeNames+=L"\""+GetTypeFullName(paths->paths[i]->pathType)+L"\""; } if(pathType) { ParsingSymbol* field=pathType->SearchClassSubSymbol(fieldName); if(!field) { errors.Add(Ptr(new ParsingError(node, L"There are multiple grammar paths with different created types get through this operation for class field \""+fieldName+L"\", but the common base type \""+GetTypeFullName(pathType)+L"\" of these types doesn't contains the required class field. Types: "+typeNames+L"; Paths: "+pathNames+L"."))); } else if(field->GetType()!=ParsingSymbol::ClassField) { errors.Add(Ptr(new ParsingError(node, L"There are multiple grammar paths with different created types get through this operation for class field \""+fieldName+L"\", and the common base type \""+GetTypeFullName(pathType)+L"\" of these types contains a symbol called \""+fieldName+L"\", but this is not a class field. Types: "+typeNames+L"; Paths: "+pathNames+L"."))); } else { return field; } } else { errors.Add(Ptr(new ParsingError(node, L"There are multiple grammar paths with different created types get through this operation for class field \""+fieldName+L"\", but these types don't have a common base type. Types: "+typeNames+L"; Paths: "+pathNames+L"."))); } return 0; } void Visit(ParsingDefinitionPrimitiveGrammar* node)override { } void Visit(ParsingDefinitionTextGrammar* node)override { } void Visit(ParsingDefinitionSequenceGrammar* node)override { } void Visit(ParsingDefinitionAlternativeGrammar* node)override { } void Visit(ParsingDefinitionLoopGrammar* node)override { } void Visit(ParsingDefinitionOptionalGrammar* node)override { } void Visit(ParsingDefinitionCreateGrammar* node)override { } void Visit(ParsingDefinitionAssignGrammar* node)override { if(ParsingSymbol* field=GetFieldFromCombined(node, node->memberName)) { manager->CacheSetSymbol(node, field); manager->CacheSetType(node, field->GetDescriptorSymbol()); ParsingSymbol* fieldType=field->GetDescriptorSymbol(); ParsingSymbol* valueType=manager->CacheGetType(node->grammar.Obj()); ParsingSymbol* targetFieldType=fieldType; if(targetFieldType->GetType()==ParsingSymbol::ArrayType) { targetFieldType=targetFieldType->GetDescriptorSymbol(); } if(targetFieldType!=valueType && valueType->SearchCommonBaseClass(targetFieldType)!=targetFieldType) { errors.Add(Ptr(new ParsingError(node, L"Cannot assign value from grammar {"+GrammarToString(node->grammar.Obj())+L"} of type \""+GetTypeFullName(valueType)+L"\" to the field \""+node->memberName+L"\" of type \""+GetTypeFullName(fieldType)+L"\"."))); } } } void Visit(ParsingDefinitionUseGrammar* node)override { } void Visit(ParsingDefinitionSetterGrammar* node)override { if(ParsingSymbol* field=GetFieldFromCombined(node, node->memberName)) { manager->CacheSetSymbol(node, field); manager->CacheSetType(node, field->GetDescriptorSymbol()); if(field->GetDescriptorSymbol()->GetType()!=ParsingSymbol::EnumType) { errors.Add(Ptr(new ParsingError(node, L"Setter operation (the \"with\" operator) can only specify the value of a class field of an enum type. But \""+GetTypeFullName(field->GetDescriptorSymbol())+L"\" is not a enum type."))); } else { ParsingSymbol* enumType=field->GetDescriptorSymbol(); ParsingSymbol* enumItem=enumType->GetSubSymbolByName(node->value); if(!enumItem) { errors.Add(Ptr(new ParsingError(node, L"Type \""+GetTypeFullName(enumType)+L"\" from field \""+node->memberName+L"\" does not have an enum item called \""+node->value+L"\"."))); } else if(enumItem->GetType()!=ParsingSymbol::EnumItem) { errors.Add(Ptr(new ParsingError(node, L"Type \""+GetTypeFullName(enumType)+L"\" from field \""+node->memberName+L"\" has a symbol called \""+node->value+L"\", but this is not an enum item."))); } } } } }; void ResolveRuleSymbols(Ptr rule, ParsingSymbolManager* manager, collections::List>& errors) { ParsingSymbol* ruleType=manager->GetGlobal()->GetSubSymbolByName(rule->name)->GetDescriptorSymbol(); for (auto grammar : rule->grammars) { List> paths; { EnumerateGrammarPathVisitor visitor(manager, rule.Obj()); grammar->Accept(&visitor); visitor.BuildPath(paths); } for (auto path : paths) { path->pathType=ruleType; vint createdTypeCount=0; vint transitionCount=0; for (auto fragment : path->fragments) { if(fragment->createdType) { createdTypeCount++; path->pathType=fragment->createdType; } if(!fragment->epsilon) { transitionCount++; } } if(createdTypeCount==0) { errors.Add(Ptr(new ParsingError(grammar.Obj(), L"No parsing tree node is created if the following path is chosen: \""+path->ToString()+L"\" in rule \""+rule->name+L"\"."))); } else if(createdTypeCount>1) { errors.Add(Ptr(new ParsingError(grammar.Obj(), L"Multiple parsing tree nodes are created if the following path is chosen: \""+path->ToString()+L"\" in rule \""+rule->name+L"\"."))); } if(transitionCount==0) { errors.Add(Ptr(new ParsingError(grammar.Obj(), L"Rule \""+rule->name+L"\" is not allowed to infer to an empty token sequence."))); } } ResolveAssignerGrammarVisitor::GrammarPathMap grammarPathMap; for (auto path : paths) { for (auto fragment : path->fragments) { ParsingDefinitionGrammar* grammar=fragment->grammar; Ptr container; vint index=grammarPathMap.Keys().IndexOf(grammar); if(index==-1) { container = Ptr(new GrammarPathContainer); grammarPathMap.Add(grammar, container); } else { container=grammarPathMap.Values().Get(index); } container->paths.Add(path); } } ResolveAssignerGrammarVisitor visitor(manager, errors, grammarPathMap); for (auto grammar : grammarPathMap.Keys()) { grammar->Accept(&visitor); } } } /*********************************************************************** ResolveSymbols ***********************************************************************/ void ResolveTypeSymbols(Ptr type, ParsingSymbolManager* manager, ParsingSymbol* scope, collections::List>& errors) { if(Ptr node=type.Cast()) { if(node->ambiguousType) { ParsingSymbol* ambigiousType=FindType(node->ambiguousType.Obj(), manager, scope, errors); WString ambiguousTypeText=TypeToString(node->ambiguousType.Obj()); if(!ambigiousType) { errors.Add(Ptr(new ParsingError(node.Obj(), L"Ambiguous type \""+ambiguousTypeText+L"\" for type \""+node->name+L"\" does not exist."))); } else if(ambigiousType->GetType()!=ParsingSymbol::ClassType) { errors.Add(Ptr(new ParsingError(node.Obj(), L"Ambiguous type \""+ambiguousTypeText+L"\" for type \""+node->name+L"\" is not a type."))); } else if(ambigiousType->GetDescriptorSymbol()!=manager->GetGlobal()->GetSubSymbolByName(node->name)) { errors.Add(Ptr(new ParsingError(node.Obj(), L"Ambiguous type \""+ambiguousTypeText+L"\" for type \""+node->name+L"\" does not inherit from \""+node->name+L"\"."))); } else { bool correct=false; if(ambigiousType->GetSubSymbolCount()==1) { ParsingSymbol* field=ambigiousType->GetSubSymbol(0); if(field->GetName()==L"items" && field->GetType()==ParsingSymbol::ClassField) { ParsingSymbol* fieldType=field->GetDescriptorSymbol(); if(fieldType->GetType()==ParsingSymbol::ArrayType && fieldType->GetDescriptorSymbol()==ambigiousType->GetDescriptorSymbol()) { correct=true; } } } if(!correct) { errors.Add(Ptr(new ParsingError(node.Obj(), L"Ambiguous type \""+ambiguousTypeText+L"\" for type \""+node->name+L"\" can only contains one field called \"item\" which should be an array of \""+node->name+L"\"."))); } } } ParsingSymbol* classType=manager->CacheGetClassType(node.Obj()); if(classType) { for (auto subType : node->subTypes) { ResolveTypeSymbols(subType, manager, classType, errors); } } } } void ResolveSymbols(Ptr definition, ParsingSymbolManager* manager, collections::List>& errors) { for (auto type : definition->types) { ResolveTypeSymbols(type, manager, manager->GetGlobal(), errors); } for (auto rule : definition->rules) { vint errorCount=errors.Count(); ValidateRuleStructure(definition, rule, manager, errors); if(errors.Count()==errorCount) { ResolveRuleSymbols(rule, manager, errors); } } } /*********************************************************************** ValidateDefinition ***********************************************************************/ void ValidateDefinition(Ptr definition, ParsingSymbolManager* manager, collections::List>& errors) { PrepareSymbols(definition, manager, errors); if(errors.Count()>0) return; ResolveSymbols(definition, manager, errors); } } } } /*********************************************************************** .\PARSINGAUTOMATON.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; namespace analyzing { /*********************************************************************** Action ***********************************************************************/ Action::Action() :actionType(Create) ,actionSource(0) ,actionTarget(0) ,creatorRule(0) ,shiftReduceSource(0) ,shiftReduceTarget(0) { } Action::~Action() { } /*********************************************************************** Transition ***********************************************************************/ Transition::Transition() :source(0) ,target(0) ,transitionType(Epsilon) ,stackOperationType(None) ,transitionSymbol(0) { } Transition::~Transition() { } bool Transition::IsEquivalent(Transition* t1, Transition* t2, bool careSourceAndTarget) { if(careSourceAndTarget) { if(t1->source!=t2->source || t1->target!=t2->target) { return false; } } if( t1->actions.Count()!=t2->actions.Count() || t1->transitionType!=t2->transitionType || t1->transitionSymbol!=t2->transitionSymbol) { return false; } for(vint j=0;jactions.Count();j++) { Ptr a1=t1->actions[j]; Ptr a2=t2->actions[j]; if( a1->actionType!=a2->actionType || a1->actionSource!=a2->actionSource || a1->actionTarget!=a2->actionTarget || a1->shiftReduceSource!=a2->shiftReduceSource ) { return false; } } return true; } /*********************************************************************** State ***********************************************************************/ State::State() :ownerRule(0) ,ownerRuleSymbol(0) ,grammarNode(0) ,stateNode(0) ,statePosition(BeforeNode) ,endState(false) { } State::~State() { } /*********************************************************************** RuleInfo ***********************************************************************/ RuleInfo::RuleInfo() :rootRuleStartState(0) ,rootRuleEndState(0) ,startState(0) ,stateNameCount(0) { } RuleInfo::~RuleInfo() { } /*********************************************************************** Automaton ***********************************************************************/ Automaton::Automaton(ParsingSymbolManager* _symbolManager) :symbolManager(_symbolManager) { } Automaton::~Automaton() { } void Automaton::AddRuleInfo(definitions::ParsingDefinitionRuleDefinition* rule, Ptr ruleInfo) { orderedRulesDefs.Add(rule); ruleInfos.Add(ruleInfo); ruleDefToInfoMap.Add(rule, ruleInfo); } State* Automaton::RuleStartState(definitions::ParsingDefinitionRuleDefinition* ownerRule) { auto state = Ptr(new State); states.Add(state); state->ownerRule=ownerRule; state->ownerRuleSymbol=symbolManager->GetGlobal()->GetSubSymbolByName(ownerRule->name); state->stateName=ownerRule->name+L".Start"; state->stateExpression=L"@ <"+ownerRule->name+L">"; return state.Obj(); } State* Automaton::RootRuleStartState(definitions::ParsingDefinitionRuleDefinition* ownerRule) { auto state = Ptr(new State); states.Add(state); state->ownerRule=ownerRule; state->ownerRuleSymbol=symbolManager->GetGlobal()->GetSubSymbolByName(ownerRule->name); state->stateName=ownerRule->name+L".RootStart"; state->stateExpression=L"@ $<"+ownerRule->name+L">"; return state.Obj(); } State* Automaton::RootRuleEndState(definitions::ParsingDefinitionRuleDefinition* ownerRule) { auto state = Ptr(new State); states.Add(state); state->ownerRule=ownerRule; state->ownerRuleSymbol=symbolManager->GetGlobal()->GetSubSymbolByName(ownerRule->name); state->stateName=ownerRule->name+L".RootEnd"; state->stateExpression=L"$<"+ownerRule->name+L"> @"; return state.Obj(); } State* Automaton::StartState(definitions::ParsingDefinitionRuleDefinition* ownerRule, definitions::ParsingDefinitionGrammar* grammarNode, definitions::ParsingDefinitionGrammar* stateNode) { auto state = Ptr(new State); states.Add(state); state->ownerRule=ownerRule; state->ownerRuleSymbol=symbolManager->GetGlobal()->GetSubSymbolByName(ownerRule->name); state->grammarNode=grammarNode; state->stateNode=stateNode; state->statePosition=State::BeforeNode; state->stateName=ownerRule->name+L"."+itow(++ruleDefToInfoMap[ownerRule]->stateNameCount); stateNode=FindAppropriateGrammarState(grammarNode, stateNode, true); state->stateExpression=L"<"+ownerRule->name+L">: "+GrammarStateToString(grammarNode, stateNode, true); return state.Obj(); } State* Automaton::EndState(definitions::ParsingDefinitionRuleDefinition* ownerRule, definitions::ParsingDefinitionGrammar* grammarNode, definitions::ParsingDefinitionGrammar* stateNode) { auto state = Ptr(new State); states.Add(state); state->ownerRule=ownerRule; state->ownerRuleSymbol=symbolManager->GetGlobal()->GetSubSymbolByName(ownerRule->name); state->grammarNode=grammarNode; state->stateNode=stateNode; state->statePosition=State::AfterNode; state->stateName=ownerRule->name+L"."+itow(++ruleDefToInfoMap[ownerRule]->stateNameCount); stateNode=FindAppropriateGrammarState(grammarNode, stateNode, false); state->stateExpression=L"<"+ownerRule->name+L">: "+GrammarStateToString(grammarNode, stateNode, false); return state.Obj(); } State* Automaton::CopyState(State* oldState) { State* resultState=0; if(oldState->statePosition==State::BeforeNode) { resultState=StartState(oldState->ownerRule, oldState->grammarNode, oldState->stateNode); } else { resultState=EndState(oldState->ownerRule, oldState->grammarNode, oldState->stateNode); } resultState->endState=oldState->endState; return resultState; } Transition* Automaton::CreateTransition(State* start, State* end) { auto transition = Ptr(new Transition); transitions.Add(transition); start->transitions.Add(transition.Obj()); end->inputs.Add(transition.Obj()); transition->source=start; transition->target=end; return transition.Obj(); } Transition* Automaton::TokenBegin(State* start, State* end) { Transition* transition=CreateTransition(start, end); transition->transitionType=Transition::TokenBegin; return transition; } Transition* Automaton::TokenFinish(State* start, State* end) { Transition* transition=CreateTransition(start, end); transition->transitionType=Transition::TokenFinish; return transition; } Transition* Automaton::NormalReduce(State* start, State* end) { Transition* transition=CreateTransition(start, end); transition->transitionType=Transition::NormalReduce; return transition; } Transition* Automaton::LeftRecursiveReduce(State* start, State* end) { Transition* transition=CreateTransition(start, end); transition->transitionType=Transition::LeftRecursiveReduce; return transition; } Transition* Automaton::Epsilon(State* start, State* end) { Transition* transition=CreateTransition(start, end); transition->transitionType=Transition::Epsilon; return transition; } Transition* Automaton::Symbol(State* start, State* end, ParsingSymbol* transitionSymbol) { Transition* transition=CreateTransition(start, end); transition->transitionType=Transition::Symbol; transition->transitionSymbol=transitionSymbol; return transition; } Transition* Automaton::CopyTransition(State* start, State* end, Transition* oldTransition) { Transition* transition=CreateTransition(start, end); transition->transitionType=oldTransition->transitionType; transition->stackOperationType=oldTransition->stackOperationType; transition->transitionSymbol=oldTransition->transitionSymbol; return transition; } void Automaton::DeleteTransition(Transition* transition) { transition->source->transitions.Remove(transition); transition->target->inputs.Remove(transition); transitions.Remove(transition); } void Automaton::DeleteState(State* state) { while(state->inputs.Count()) { DeleteTransition(state->inputs[0]); } while(state->transitions.Count()) { DeleteTransition(state->transitions[0]); } states.Remove(state); } } } } /*********************************************************************** .\PARSINGAUTOMATON_CLOSURE.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; namespace analyzing { /*********************************************************************** CreateNondeterministicPDAFromEpsilonPDA::closure_searching ***********************************************************************/ // closure function for searching reachable states ClosureItem::SearchResult EpsilonClosure(Transition* transition) { return transition->transitionType!=Transition::Epsilon?ClosureItem::Hit: //transition->target->endState?ClosureItem::Blocked: ClosureItem::Continue; } // closure searching function void SearchClosureInternal(ClosureItem::SearchResult(*closurePredicate)(Transition*), List& transitionPath, Transition* transition, State* state, List& closure) { for (auto singleTransitionPath : transitionPath) { if(singleTransitionPath->source==state && closurePredicate(singleTransitionPath)!=ClosureItem::Blocked) { auto path = Ptr(new List); CopyFrom(*path.Obj(), transitionPath); closure.Add(ClosureItem(state, path, true)); return; } } ClosureItem::SearchResult result=transition?closurePredicate(transition):ClosureItem::Continue; switch(result) { case ClosureItem::Continue: { for (auto newTransition : state->transitions) { if(!transitionPath.Contains(newTransition)) { transitionPath.Add(newTransition); SearchClosureInternal(closurePredicate, transitionPath, newTransition, newTransition->target, closure); transitionPath.RemoveAt(transitionPath.Count()-1); } } } break; case ClosureItem::Hit: { auto path = Ptr(new List); CopyFrom(*path.Obj(), transitionPath); closure.Add(ClosureItem(state, path, false)); } break; default:; } } void SearchClosure(ClosureItem::SearchResult(*closurePredicate)(Transition*), State* startState, List& closure) { List transitionPath; SearchClosureInternal(closurePredicate, transitionPath, 0, startState, closure); } // map old state to new state and track all states that are not visited yet State* GetMappedState(Ptr newAutomaton, State* oldState, List& scanningStates, Dictionary& oldNewStateMap) { State* newState=0; vint mapIndex=oldNewStateMap.Keys().IndexOf(oldState); if(mapIndex==-1) { newState=newAutomaton->CopyState(oldState); oldNewStateMap.Add(oldState, newState); } else { newState=oldNewStateMap.Values().Get(mapIndex); } if(!scanningStates.Contains(oldState)) { scanningStates.Add(oldState); } return newState; } /*********************************************************************** RemoveEpsilonTransitions ***********************************************************************/ void RemoveEpsilonTransitions(collections::Dictionary& oldNewStateMap, collections::List& scanningStates, Ptr automaton) { vint currentStateIndex=0; while(currentStateIndex closure; SearchClosure(&EpsilonClosure, currentOldState, closure); for (auto closureItem : closure) { Transition* oldTransition=closureItem.transitions->Get(closureItem.transitions->Count()-1); if(!closureItem.cycle || oldTransition->transitionType!=Transition::Epsilon) { // if the oldTransition begins from an end state if(oldTransition->source->endState && closureItem.transitions->Count()>1) { // keep a epsilon transition that without the last "TokenFinish" State* newEndState=GetMappedState(automaton, oldTransition->source, scanningStates, oldNewStateMap); Transition* transition=automaton->Epsilon(currentNewState, newEndState); for (auto pathTransition : *closureItem.transitions.Obj()) { if(pathTransition==oldTransition) break; CopyFrom(transition->actions, pathTransition->actions, true); } } else { // build compacted non-epsilon transition to the target state of the path State* newEndState=GetMappedState(automaton, oldTransition->target, scanningStates, oldNewStateMap); Transition* transition=automaton->CopyTransition(currentNewState, newEndState, oldTransition); for (auto pathTransition : *closureItem.transitions.Obj()) { CopyFrom(transition->actions, pathTransition->actions, true); } } } } } } } } } /*********************************************************************** .\PARSINGAUTOMATON_EPDA.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; namespace analyzing { /*********************************************************************** CreateEpsilonPDAVisitor ***********************************************************************/ class CreateEpsilonPDAVisitor : public Object, public ParsingDefinitionGrammar::IVisitor { public: Ptr automaton; ParsingDefinitionRuleDefinition* rule; ParsingDefinitionGrammar* ruleGrammar; State* startState; State* endState; Transition* result; CreateEpsilonPDAVisitor(Ptr _automaton, ParsingDefinitionRuleDefinition* _rule, ParsingDefinitionGrammar* _ruleGrammar, State* _startState, State* _endState) :automaton(_automaton) ,rule(_rule) ,ruleGrammar(_ruleGrammar) ,startState(_startState) ,endState(_endState) ,result(0) { } static Transition* Create(ParsingDefinitionGrammar* grammar, Ptr automaton, ParsingDefinitionRuleDefinition* rule, ParsingDefinitionGrammar* ruleGrammar, State* startState, State* endState) { CreateEpsilonPDAVisitor visitor(automaton, rule, ruleGrammar, startState, endState); grammar->Accept(&visitor); return visitor.result; } Transition* Create(ParsingDefinitionGrammar* grammar, State* startState, State* endState) { return Create(grammar, automaton, rule, ruleGrammar, startState, endState); } void Visit(ParsingDefinitionPrimitiveGrammar* node)override { result=automaton->Symbol(startState, endState, automaton->symbolManager->CacheGetSymbol(node)); } void Visit(ParsingDefinitionTextGrammar* node)override { result=automaton->Symbol(startState, endState, automaton->symbolManager->CacheGetSymbol(node)); } void Visit(ParsingDefinitionSequenceGrammar* node)override { State* middleState=automaton->EndState(startState->ownerRule, ruleGrammar, node->first.Obj()); Create(node->first.Obj(), startState, middleState); Create(node->second.Obj(), middleState, endState); } void Visit(ParsingDefinitionAlternativeGrammar* node)override { Create(node->first.Obj(), startState, endState); Create(node->second.Obj(), startState, endState); } void Visit(ParsingDefinitionLoopGrammar* node)override { State* loopStart=automaton->StartState(startState->ownerRule, ruleGrammar, node->grammar.Obj()); automaton->Epsilon(startState, loopStart); automaton->Epsilon(loopStart, endState); Create(node->grammar.Obj(), loopStart, loopStart); } void Visit(ParsingDefinitionOptionalGrammar* node)override { Create(node->grammar.Obj(), startState, endState); automaton->Epsilon(startState, endState); } void Visit(ParsingDefinitionCreateGrammar* node)override { State* middleState=automaton->EndState(startState->ownerRule, ruleGrammar, node->grammar.Obj()); Create(node->grammar.Obj(), startState, middleState); Transition* transition=automaton->Epsilon(middleState, endState); auto action = Ptr(new Action); action->actionType=Action::Create; action->actionSource=automaton->symbolManager->CacheGetType(node->type.Obj(), 0); action->creatorRule=rule; transition->actions.Add(action); } void Visit(ParsingDefinitionAssignGrammar* node)override { Transition* transition=Create(node->grammar.Obj(), startState, endState); auto action = Ptr(new Action); action->actionType=Action::Assign; action->actionSource=automaton->symbolManager->CacheGetSymbol(node); action->creatorRule=rule; transition->actions.Add(action); } void Visit(ParsingDefinitionUseGrammar* node)override { Transition* transition=Create(node->grammar.Obj(), startState, endState); auto action = Ptr(new Action); action->actionType=Action::Using; action->creatorRule=rule; transition->actions.Add(action); } void Visit(ParsingDefinitionSetterGrammar* node)override { State* middleState=automaton->EndState(startState->ownerRule, ruleGrammar, node->grammar.Obj()); Create(node->grammar.Obj(), startState, middleState); Transition* transition=automaton->Epsilon(middleState, endState); auto action = Ptr(new Action); action->actionType=Action::Setter; action->actionSource=automaton->symbolManager->CacheGetSymbol(node); action->actionTarget=action->actionSource->GetDescriptorSymbol()->GetSubSymbolByName(node->value); action->creatorRule=rule; transition->actions.Add(action); } }; /*********************************************************************** CreateRuleEpsilonPDA ***********************************************************************/ void CreateRuleEpsilonPDA(Ptr automaton, Ptr rule, ParsingSymbolManager* manager) { auto ruleInfo = Ptr(new RuleInfo); automaton->AddRuleInfo(rule.Obj(), ruleInfo); ruleInfo->rootRuleStartState=automaton->RootRuleStartState(rule.Obj()); ruleInfo->rootRuleEndState=automaton->RootRuleEndState(rule.Obj()); ruleInfo->startState=automaton->RuleStartState(rule.Obj()); automaton->TokenBegin(ruleInfo->rootRuleStartState, ruleInfo->startState); for (auto grammar : rule->grammars) { State* grammarStartState=automaton->StartState(rule.Obj(), grammar.Obj(), grammar.Obj()); State* grammarEndState=automaton->EndState(rule.Obj(), grammar.Obj(), grammar.Obj()); grammarEndState->stateName+=L".End"; grammarEndState->endState=true; automaton->Epsilon(ruleInfo->startState, grammarStartState); automaton->TokenFinish(grammarEndState, ruleInfo->rootRuleEndState); ruleInfo->endStates.Add(grammarEndState); CreateEpsilonPDAVisitor::Create(grammar.Obj(), automaton, rule.Obj(), grammar.Obj(), grammarStartState, grammarEndState); } } /*********************************************************************** CreateEpsilonPDA ***********************************************************************/ Ptr CreateEpsilonPDA(Ptr definition, ParsingSymbolManager* manager) { auto automaton = Ptr(new Automaton(manager)); for (auto rule : definition->rules) { CreateRuleEpsilonPDA(automaton, rule, manager); } return automaton; } } } } /*********************************************************************** .\PARSINGAUTOMATON_GENERATETABLE.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; using namespace tabling; namespace analyzing { /*********************************************************************** GetTypeNameForCreateInstruction ***********************************************************************/ WString GetTypeNameForCreateInstruction(ParsingSymbol* type) { ParsingSymbol* parent=type->GetParentSymbol(); if(parent->GetType()==ParsingSymbol::ClassType) { return GetTypeNameForCreateInstruction(parent)+L"."+type->GetName(); } else { return type->GetName(); } } /*********************************************************************** CreateLookAhead ***********************************************************************/ void CopyStableLookAheads(List>& la, List>& sla, const List>& la2) { CopyFrom(sla, From(la) .Where([&](auto&& lai) { return From(la2).All([&](auto&& lai2) { return ParsingTable::LookAheadInfo::TestPrefix(lai, lai2)==ParsingTable::LookAheadInfo::NotPrefix; }); }), true); } void RemoveStableLookAheads(List>& la, const List>& sla) { for(vint i=la.Count()-1;i>=0;i--) { if(sla.Contains(la[i].Obj())) { la.RemoveAt(i); } } } bool WalkLookAheads(Ptr table, List>& la, vint maxTokenCount) { vint count=la.Count(); for(vint i=0;i lai=la[i]; if(lai->tokens.Count()==maxTokenCount) { return false; } ParsingTable::LookAheadInfo::Walk(table, lai, lai->state, la); } return true; } void CompactLookAheads(Ptr t, List>& sla) { CopyFrom(sla, t->lookAheads, true); CopyFrom(t->lookAheads, From(sla) .Where([&](auto&& lai) { return From(sla).All([&](auto&& lai2) { if(lai==lai2) return true; ParsingTable::LookAheadInfo::PrefixResult result=ParsingTable::LookAheadInfo::TestPrefix(lai, lai2); switch(result) { case ParsingTable::LookAheadInfo::Prefix: return false; case ParsingTable::LookAheadInfo::Equal: return sla.IndexOf(lai.Obj()) < sla.IndexOf(lai2.Obj()); default: return true; } }); })); } bool CreateLookAhead(Ptr table, Ptr t1, Ptr t2, vint maxTokenCount) { List> la1, la2, sla1, sla2; // calculate 1-token look aheads ParsingTable::LookAheadInfo::Walk(table, 0, t1->targetState, la1); ParsingTable::LookAheadInfo::Walk(table, 0, t2->targetState, la2); do { // pick up all stable look aheads and remove them from the look ahead list // stable look ahead means, when the look ahead is satisfied, then the transition is picked up with full confidence // non-stable look ahead means, when the look ahead is satisifed, it only increase confidence, needs further tokens for decision CopyStableLookAheads(la1, sla1, la2); CopyStableLookAheads(la2, sla2, la1); RemoveStableLookAheads(la1, sla1); RemoveStableLookAheads(la2, sla2); // check if there are non-stable look aheads in two transitions points to the same state // in such situation means that the two transition cannot always be determined using look aheads for (auto lai1 : la1) { for (auto lai2 : la2) { if (lai1->state == lai2->state) { if (ParsingTable::LookAheadInfo::TestPrefix(lai1, lai2) != ParsingTable::LookAheadInfo::NotPrefix) { return false; } if (ParsingTable::LookAheadInfo::TestPrefix(lai2, lai1) != ParsingTable::LookAheadInfo::NotPrefix) { return false; } } } } // use the non-stable look aheads to walk a token further if(!WalkLookAheads(table, la1, maxTokenCount) || !WalkLookAheads(table, la2, maxTokenCount)) { return false; } } while(la1.Count()>0 || la2.Count()>0); CompactLookAheads(t1, sla1); CompactLookAheads(t2, sla2); return true; } /*********************************************************************** CollectAttribute ***********************************************************************/ void CollectType(ParsingSymbol* symbol, List& types) { if(symbol->GetType()==ParsingSymbol::ClassType) { types.Add(symbol); } vint count=symbol->GetSubSymbolCount(); for(vint i=0;iGetSubSymbol(i), types); } } void CollectAttributeInfo(Ptr att, List>& atts) { for (auto datt : atts) { auto tatt = Ptr(new ParsingTable::AttributeInfo(datt->name)); CopyFrom(tatt->arguments, datt->arguments); att->attributes.Add(tatt); } } Ptr CreateAttributeInfo(List>& atts) { auto att = Ptr(new ParsingTable::AttributeInfoList); CollectAttributeInfo(att, atts); return att; } /*********************************************************************** GenerateTable ***********************************************************************/ vint LookAheadConflictPriority(vint tableTokenIndex) { switch (tableTokenIndex) { case ParsingTable::NormalReduce: return 0; case ParsingTable::LeftRecursiveReduce: return 1; default: return 2; } } void GenerateLookAhead(Ptr table, List& stateIds, vint state, vint token, Ptr t1, Ptr t2, bool enableAmbiguity, collections::List>& errors) { if(ParsingTable::TransitionItem::CheckOrder(t1, t2, ParsingTable::TransitionItem::UnknownOrder)==ParsingTable::TransitionItem::UnknownOrder) { if(enableAmbiguity || !CreateLookAhead(table, t1, t2, 16)) { if (LookAheadConflictPriority(t1->token) != LookAheadConflictPriority(t2->token)) { return; } WString stateName=itow(state)+L"["+table->GetStateInfo(state).stateName+L"]"; WString tokenName= token==ParsingTable::TokenBegin?WString(L"$TokenBegin"): token==ParsingTable::TokenFinish?WString(L"$TokenFinish"): token==ParsingTable::NormalReduce?WString(L"$NormalReduce"): token==ParsingTable::LeftRecursiveReduce?WString(L"$LeftRecursiveReduce"): table->GetTokenInfo(token).name; switch (t1->token) { case ParsingTable::NormalReduce: errors.Add(Ptr(new ParsingError(stateIds[state]->ownerRule, L"Conflict happened with normal reduce in transition of \"" + tokenName + L"\" of state \"" + stateName + L"\"."))); break; case ParsingTable::LeftRecursiveReduce: errors.Add(Ptr(new ParsingError(stateIds[state]->ownerRule, L"Conflict happened with left recursive reduce in transition of \"" + tokenName + L"\" of state \"" + stateName + L"\"."))); break; default: errors.Add(Ptr(new ParsingError(stateIds[state]->ownerRule, L"Conflict happened in transition of \"" + tokenName + L"\" of state \"" + stateName + L"\"."))); break; } } } } Ptr GenerateTableFromPDA(Ptr definition, ParsingSymbolManager* manager, Ptr jointPDA, bool enableAmbiguity, collections::List>& errors) { List> atts; /*********************************************************************** find all class types ***********************************************************************/ List types; Dictionary typeAtts; Dictionary, vint> treeFieldAtts; // stable class field order List orderedChildTypeKeys; Dictionary>> childTypeValues; // find all class types CollectType(manager->GetGlobal(), types); for (auto type : types) { auto typeAtt = Ptr(new ParsingTable::AttributeInfoList); ParsingSymbol* parent = type; while (parent) { ParsingDefinitionClassDefinition* classDef = manager->CacheGetClassDefinition(parent); CollectAttributeInfo(typeAtt, classDef->attributes); Ptr> children; vint index = childTypeValues.Keys().IndexOf(parent); if (index == -1) { children = Ptr(new List); orderedChildTypeKeys.Add(parent); childTypeValues.Add(parent, children); } else { children = childTypeValues.Values().Get(index); } children->Add(type); parent = parent->GetDescriptorSymbol(); } if (typeAtt->attributes.Count() > 0) { typeAtts.Add(GetTypeFullName(type), atts.Count()); atts.Add(typeAtt); } else { typeAtts.Add(GetTypeFullName(type), -1); } } // find all class fields for (auto type : orderedChildTypeKeys) { List& children = *childTypeValues[type].Obj(); ParsingDefinitionClassDefinition* classDef = manager->CacheGetClassDefinition(type); List fieldAtts; for (auto [field, index] : indexed(classDef->members)) { if (field->attributes.Count() > 0) { fieldAtts.Add(atts.Count()); atts.Add(CreateAttributeInfo(field->attributes)); } else { fieldAtts.Add(-1); } } for (auto child : children) { WString type = GetTypeFullName(child); for (auto [field, index] : indexed(classDef->members)) { treeFieldAtts.Add(Pair(type, field->name), fieldAtts[index]); } } } /*********************************************************************** find all tokens ***********************************************************************/ vint tokenCount = 0; vint discardTokenCount = 0; Dictionary tokenIds; List discardTokens; Dictionary tokenAtts; Dictionary ruleAtts; for (auto token : definition->tokens) { if (token->attributes.Count() > 0) { tokenAtts.Add(token->name, atts.Count()); atts.Add(CreateAttributeInfo(token->attributes)); } else { tokenAtts.Add(token->name, -1); } if (token->discard) { discardTokens.Add(token->name); discardTokenCount++; } else { ParsingSymbol* tokenSymbol = jointPDA->symbolManager->GetGlobal()->GetSubSymbolByName(token->name); tokenIds.Add(tokenSymbol, tokenIds.Count() + ParsingTable::UserTokenStart); tokenCount++; } } /*********************************************************************** find all rules ***********************************************************************/ for (auto rule : definition->rules) { if (rule->attributes.Count() > 0) { ruleAtts.Add(rule->name, atts.Count()); atts.Add(CreateAttributeInfo(rule->attributes)); } else { ruleAtts.Add(rule->name, -1); } } /*********************************************************************** find all available states ***********************************************************************/ List stateIds; vint availableStateCount = 0; { vint currentState = 0; List scanningStates; for (auto ruleInfo : jointPDA->ruleInfos) { if (!scanningStates.Contains(ruleInfo->rootRuleStartState)) { scanningStates.Add(ruleInfo->rootRuleStartState); } while (currentState < scanningStates.Count()) { State* state = scanningStates[currentState++]; stateIds.Add(state); for (auto transition : state->transitions) { if (!scanningStates.Contains(transition->target)) { scanningStates.Add(transition->target); } } } } availableStateCount = scanningStates.Count(); } // there will be some states that is used in shift and reduce but it is not a reachable state // so the state table will record all state for (auto state : jointPDA->states) { if (!stateIds.Contains(state.Obj())) { stateIds.Add(state.Obj()); } } vint stateCount = stateIds.Count(); auto table = Ptr(new ParsingTable(atts.Count(), typeAtts.Count(), treeFieldAtts.Count(), tokenCount, discardTokenCount, stateCount, definition->rules.Count())); /*********************************************************************** fill attribute infos ***********************************************************************/ for (auto [att, index] : indexed(atts)) { table->SetAttributeInfo(index, att); } /*********************************************************************** fill tree type infos ***********************************************************************/ typedef Pair TreeTypeAttsPair; for (auto [type, index] : indexed(typeAtts)) { table->SetTreeTypeInfo(index, ParsingTable::TreeTypeInfo(type.key, type.value)); } /*********************************************************************** fill tree field infos ***********************************************************************/ typedef Pair, vint> TreeFieldAttsPair; for (auto [field, index] : indexed(treeFieldAtts)) { table->SetTreeFieldInfo(index, ParsingTable::TreeFieldInfo(field.key.key, field.key.value, field.value)); } /*********************************************************************** fill token infos ***********************************************************************/ for (auto symbol : tokenIds.Keys()) { ParsingTable::TokenInfo info; info.name = symbol->GetName(); info.regex = symbol->GetDescriptorString(); info.attributeIndex = tokenAtts[info.name]; vint id = tokenIds[symbol]; table->SetTokenInfo(id, info); } for (auto [name, i] : indexed(discardTokens)) { ParsingSymbol* symbol = jointPDA->symbolManager->GetGlobal()->GetSubSymbolByName(name); ParsingTable::TokenInfo info; info.name = symbol->GetName(); info.regex = symbol->GetDescriptorString(); info.attributeIndex = tokenAtts[info.name]; table->SetDiscardTokenInfo(i, info); } /*********************************************************************** fill rule infos ***********************************************************************/ for (auto [rule, i] : indexed(jointPDA->orderedRulesDefs)) { Ptr pdaRuleInfo = jointPDA->ruleDefToInfoMap[rule]; ParsingTable::RuleInfo info; info.name = rule->name; info.type = TypeToString(rule->type.Obj()); info.rootStartState = stateIds.IndexOf(pdaRuleInfo->rootRuleStartState); info.attributeIndex = ruleAtts[info.name]; if (Ptr classType = rule->type.Cast()) { ParsingSymbol* ruleSymbol = manager->GetGlobal()->GetSubSymbolByName(rule->name); ParsingSymbol* ruleType = ruleSymbol->GetDescriptorSymbol(); ParsingDefinitionClassDefinition* ruleTypeDef = manager->CacheGetClassDefinition(ruleType); if (ruleTypeDef && ruleTypeDef->ambiguousType) { ParsingSymbol* ambiguousType = manager->CacheGetType(ruleTypeDef->ambiguousType.Obj(), ruleType->GetParentSymbol()); info.ambiguousType = GetTypeFullName(ambiguousType); } } table->SetRuleInfo(i, info); } /*********************************************************************** fill state infos ***********************************************************************/ for (auto [state, i] : indexed(stateIds)) { ParsingTable::StateInfo info; info.ruleName = state->ownerRule->name; info.stateName = state->stateName; info.stateExpression = state->stateExpression; table->SetStateInfo(i, info); } /*********************************************************************** fill transition table ***********************************************************************/ for (auto [state, stateIndex] : indexed(stateIds)) { // if this state is not necessary, stop building the table if (stateIndex >= availableStateCount) break; for (auto transition : state->transitions) { vint tokenIndex = -1; switch (transition->transitionType) { case Transition::TokenBegin: tokenIndex = ParsingTable::TokenBegin; break; case Transition::TokenFinish: tokenIndex = ParsingTable::TokenFinish; break; case Transition::NormalReduce: tokenIndex = ParsingTable::NormalReduce; break; case Transition::LeftRecursiveReduce: tokenIndex = ParsingTable::LeftRecursiveReduce; break; case Transition::Symbol: tokenIndex = tokenIds[transition->transitionSymbol]; break; default:; } Ptr bag = table->GetTransitionBag(stateIndex, tokenIndex); if (!bag) { bag = Ptr(new ParsingTable::TransitionBag); table->SetTransitionBag(stateIndex, tokenIndex, bag); } auto item = Ptr(new ParsingTable::TransitionItem); item->token = tokenIndex; item->targetState = stateIds.IndexOf(transition->target); bag->transitionItems.Add(item); for (auto action : transition->actions) { ParsingTable::Instruction ins; switch (action->actionType) { case Action::Create: { ins.instructionType = ParsingTable::Instruction::Create; ins.nameParameter = GetTypeNameForCreateInstruction(action->actionSource); } break; case Action::Using: { ins.instructionType = ParsingTable::Instruction::Using; } break; case Action::Assign: { if (action->actionSource->GetDescriptorSymbol()->GetType() == ParsingSymbol::ArrayType) { ins.instructionType = ParsingTable::Instruction::Item; } else { ins.instructionType = ParsingTable::Instruction::Assign; } ins.nameParameter = action->actionSource->GetName(); } break; case Action::Setter: { ins.instructionType = ParsingTable::Instruction::Setter; ins.nameParameter = action->actionSource->GetName(); ins.value = action->actionTarget->GetName(); } break; case Action::Shift: { ins.instructionType = ParsingTable::Instruction::Shift; ins.stateParameter = stateIds.IndexOf(action->shiftReduceSource); } break; case Action::Reduce: { ins.instructionType = ParsingTable::Instruction::Reduce; ins.stateParameter = stateIds.IndexOf(action->shiftReduceSource); item->stackPattern.Add(ins.stateParameter); } break; case Action::LeftRecursiveReduce: { ins.instructionType = ParsingTable::Instruction::LeftRecursiveReduce; ins.stateParameter = stateIds.IndexOf(action->shiftReduceSource); } break; } ins.creatorRule = action->creatorRule->name; item->instructions.Add(ins); } } } /*********************************************************************** check conflict and build look ahead table ***********************************************************************/ for (vint i = 0; i < table->GetStateCount(); i++) { for (vint j = 0; j < table->GetTokenCount(); j++) { Ptr bag = table->GetTransitionBag(i, j); if (bag) { CopyFrom( bag->transitionItems, From(bag->transitionItems) .OrderBy([&](auto&& t1, auto&& t2) { // stable transition order vint i1 = bag->transitionItems.IndexOf(t1.Obj()); vint i2 = bag->transitionItems.IndexOf(t2.Obj()); auto defaultOrder = i1 < i2 ? ParsingTable::TransitionItem::CorrectOrder : i1 > i2 ? ParsingTable::TransitionItem::WrongOrder : ParsingTable::TransitionItem::SameOrder ; return ParsingTable::TransitionItem::Compare(t1, t2, defaultOrder) <=> 0; })); // build look ahead inside a transition for (vint k1 = 0; k1 < bag->transitionItems.Count() - 1; k1++) { for (vint k2 = k1 + 1; k2 < bag->transitionItems.Count(); k2++) { Ptr t1 = bag->transitionItems[k1]; Ptr t2 = bag->transitionItems[k2]; GenerateLookAhead(table, stateIds, i, j, t1, t2, enableAmbiguity, errors); } } // build look ahead between this transition and reduce transitions for (vint t = ParsingTable::NormalReduce; t <= ParsingTable::LeftRecursiveReduce && t < j; t++) { if (Ptr reduceBag = table->GetTransitionBag(i, t)) { for (vint k1 = 0; k1 < reduceBag->transitionItems.Count(); k1++) { for (vint k2 = 0; k2 < bag->transitionItems.Count(); k2++) { Ptr t1 = reduceBag->transitionItems[k1]; Ptr t2 = bag->transitionItems[k2]; GenerateLookAhead(table, stateIds, i, j, t1, t2, enableAmbiguity, errors); } } } } } } } /*********************************************************************** initialize table ***********************************************************************/ if (errors.Count() > 0) { table->SetAmbiguity(true); } table->Initialize(); return table; } Ptr GenerateTable(Ptr definition, bool enableAmbiguity, collections::List>& errors) { errors.Clear(); ParsingSymbolManager symbolManager; ValidateDefinition(definition, &symbolManager, errors); if(errors.Count()==0) { Ptr epsilonPDA=CreateEpsilonPDA(definition, &symbolManager); Ptr nondeterministicPDA=CreateNondeterministicPDAFromEpsilonPDA(epsilonPDA); Ptr jointPDA=CreateJointPDAFromNondeterministicPDA(nondeterministicPDA); CompactJointPDA(jointPDA); MarkLeftRecursiveInJointPDA(jointPDA, errors); if(errors.Count()==0) { Ptr table=GenerateTableFromPDA(definition, &symbolManager, jointPDA, enableAmbiguity, errors); if(enableAmbiguity || errors.Count()==0) { return table; } } } return 0; } } } } /*********************************************************************** .\PARSINGAUTOMATON_JPDA.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; namespace analyzing { /*********************************************************************** CreateJointPDAFromNondeterministicPDA ***********************************************************************/ Ptr CreateJointPDAFromNondeterministicPDA(Ptr nondeterministicPDA) { auto automaton = Ptr(new Automaton(nondeterministicPDA->symbolManager)); // build rule info data Dictionary ruleMap; Dictionary oldNewStateMap; for (auto rule : nondeterministicPDA->orderedRulesDefs) { // build new rule info auto ruleInfo = nondeterministicPDA->ruleDefToInfoMap[rule]; auto newRuleInfo = Ptr(new RuleInfo); automaton->AddRuleInfo(rule, newRuleInfo); ruleMap.Add(rule->name, rule); newRuleInfo->rootRuleStartState=automaton->RootRuleStartState(rule); newRuleInfo->rootRuleEndState=automaton->RootRuleEndState(rule); newRuleInfo->startState=automaton->RuleStartState(rule); oldNewStateMap.Add(ruleInfo->rootRuleStartState, newRuleInfo->rootRuleStartState); oldNewStateMap.Add(ruleInfo->rootRuleEndState, newRuleInfo->rootRuleEndState); oldNewStateMap.Add(ruleInfo->startState, newRuleInfo->startState); newRuleInfo->rootRuleStartState->stateExpression=ruleInfo->rootRuleStartState->stateExpression; newRuleInfo->rootRuleEndState->stateExpression=ruleInfo->rootRuleEndState->stateExpression; newRuleInfo->startState->stateExpression=ruleInfo->startState->stateExpression; } for (auto oldState : nondeterministicPDA->states) { if((oldState->inputs.Count()>0 || oldState->transitions.Count()>0) && !oldNewStateMap.Keys().Contains(oldState.Obj())) { State* newState=automaton->CopyState(oldState.Obj()); oldNewStateMap.Add(oldState.Obj(), newState); newState->stateExpression=oldState->stateExpression; } } // create transitions for (auto rule : nondeterministicPDA->orderedRulesDefs) { Ptr ruleInfo=nondeterministicPDA->ruleDefToInfoMap[rule]; Ptr newRuleInfo=automaton->ruleDefToInfoMap[rule]; // complete new rule info for (auto endState : ruleInfo->endStates) { newRuleInfo->endStates.Add(oldNewStateMap[endState]); } // create joint transitions according to old automaton List scanningStates; vint currentStateIndex=0; scanningStates.Add(ruleInfo->rootRuleStartState); while(currentStateIndextransitions) { State* oldSource=oldTransition->source; State* oldTarget=oldTransition->target; State* newSource=oldNewStateMap[oldSource]; State* newTarget=oldNewStateMap[oldTarget]; if(!scanningStates.Contains(oldSource)) scanningStates.Add(oldSource); if(!scanningStates.Contains(oldTarget)) scanningStates.Add(oldTarget); if(oldTransition->transitionType==Transition::Symbol && oldTransition->transitionSymbol->GetType()==ParsingSymbol::RuleDef) { // if this is a rule transition, create // source -> ruleStart // ruleEnd[] -> target ParsingDefinitionRuleDefinition* rule=ruleMap[oldTransition->transitionSymbol->GetName()]; Ptr oldRuleInfo=nondeterministicPDA->ruleDefToInfoMap[rule]; { Transition* shiftTransition=automaton->Epsilon(newSource, oldNewStateMap[oldRuleInfo->startState]); auto action = Ptr(new Action); action->actionType=Action::Shift; action->shiftReduceSource=newSource; action->shiftReduceTarget=newTarget; action->creatorRule=shiftTransition->source->ownerRule; shiftTransition->actions.Add(action); } for (auto oldEndState : oldRuleInfo->endStates) { Transition* reduceTransition=automaton->NormalReduce(oldNewStateMap[oldEndState], newTarget); auto action = Ptr(new Action); action->actionType=Action::Reduce; action->shiftReduceSource=newSource; action->shiftReduceTarget=newTarget; action->creatorRule=reduceTransition->source->ownerRule; reduceTransition->actions.Add(action); CopyFrom(reduceTransition->actions, oldTransition->actions, true); } } else { // if not, just copy Transition* newTransition=automaton->CopyTransition(newSource, newTarget, oldTransition); CopyFrom(newTransition->actions, oldTransition->actions); } } } } return automaton; } /*********************************************************************** CompactJointPDA ***********************************************************************/ // closure function for searching shift-reduce-compact transition ClosureItem::SearchResult ShiftReduceCompactClosure(Transition* transition) { return transition->stackOperationType!=Transition::None?ClosureItem::Blocked: transition->transitionType!=Transition::Epsilon?ClosureItem::Hit: ClosureItem::Continue; } void CompactJointPDA(Ptr jointPDA) { for (auto state : jointPDA->states) { State* currentState=state.Obj(); // search for epsilon closure List closure; SearchClosure(&ShiftReduceCompactClosure, currentState, closure); for (auto closureItem : closure) { Transition* lastTransition=closureItem.transitions->Get(closureItem.transitions->Count()-1); Transition::StackOperationType stackOperationType=Transition::None; Transition::TransitionType transitionType=lastTransition->transitionType; if(closureItem.cycle && lastTransition->transitionType==Transition::Epsilon) { bool containsShift=false; bool containsReduce=false; for (auto pathTransition : *closureItem.transitions.Obj()) { for (auto action : pathTransition->actions) { if(action->actionType==Action::Shift) containsShift=true; if(action->actionType==Action::Reduce) containsReduce=true; } } if(containsShift && !containsReduce) { // a left recursive compacted shift transition is found // if the left recursive state is not the current state // that means this transition path fall into other left recursive state // e.g. // Term = Factor | Term (here is a left recursion) * Factor // Exp = Term (this rule symbol transition will fall into Term's left recursive state) ... // if such a case happened, this transition path will be simply discarded if(closureItem.state==currentState) { stackOperationType=Transition::LeftRecursive; } } else if(!containsShift && containsReduce) { // a right recursive compacted reduce transition is found // if this state will receive $TokenFinish, then the stack pattern number can be infinite // e.g. for right recursive expression "a b c" == "(a (b c))" // when trying to do a transition by $TokenFinish // "a b" should reduce once // "a b c" should reduce twice // because that a reduce is not considered a virtual token, so this is not going to be happened } } else if(closureItem.transitions->Count()>1) { // in joint PDA, only shift and reduce transitions are epsilon transition // if there are more than one transition in a path, then there should be shift or reduce transitions in the path stackOperationType=Transition::ShiftReduceCompacted; } if(stackOperationType!=Transition::None) { // build shift-reduce-compacted transition to the target state of the path Transition* transition=jointPDA->CopyTransition(currentState, lastTransition->target, lastTransition); transition->transitionType=transitionType; transition->stackOperationType=stackOperationType; // there will be , or // but there will not be something like // so we can append stackPattern safely for (auto pathTransition : *closureItem.transitions.Obj()) { CopyFrom(transition->actions, pathTransition->actions, true); } } } } // delete unnecessary transactions for(vint i=jointPDA->transitions.Count()-1;i>=0;i--) { Transition* transition=jointPDA->transitions[i].Obj(); if(transition->stackOperationType==Transition::None && transition->transitionType==Transition::Epsilon) { jointPDA->DeleteTransition(transition); } } } /*********************************************************************** MarkLeftRecursiveInJointPDA ***********************************************************************/ void MarkLeftRecursiveInJointPDA(Ptr jointPDA, collections::List>& errors) { vint errorCount=errors.Count(); // record all left recursive shifts and delete all left recursive epsilon transition SortedList> leftRecursiveShifts; for (auto state : jointPDA->states) { for(vint i=state->transitions.Count()-1;i>=0;i--) { Transition* transition=state->transitions[i]; if(transition->stackOperationType==Transition::LeftRecursive) { Ptr shiftAction; for (auto action : transition->actions) { if(action->actionType==Action::Shift) { if(shiftAction) { errors.Add(Ptr(new ParsingError(state->ownerRule, L"Indirect left recursive transition in rule \"" + state->ownerRule->name + L"\" is not allowed."))); goto FOUND_INDIRECT_LEFT_RECURSIVE_TRANSITION; } else { shiftAction=action; } } } if(shiftAction) { leftRecursiveShifts.Add(Pair(shiftAction->shiftReduceSource, shiftAction->shiftReduceTarget)); } FOUND_INDIRECT_LEFT_RECURSIVE_TRANSITION: jointPDA->DeleteTransition(transition); } } } if(errorCount!=errors.Count()) { return; } // change all reduce actions whose (shiftReduceSource, shiftReduceTarget) is recorded in leftRecursiveShifts to left-recursive-reduce // when a reduce is converted to a left-recursive-reduce, the corresponding state in stackPattern should be removed // so this will keep count(Reduce) == count(stackPattern) for (auto state : jointPDA->states) { for (auto transition : state->transitions) { for(vint i=transition->actions.Count()-1;i>=0;i--) { Ptr action=transition->actions[i]; if(action->actionType==Action::Reduce) { Pair shift(action->shiftReduceSource, action->shiftReduceTarget); if(leftRecursiveShifts.Contains(shift)) { // check if this is a normal reduce transition, and change it to a left recursive reduce transition. if (transition->transitionType == Transition::NormalReduce) { transition->transitionType = Transition::LeftRecursiveReduce; // need to create a new action because in the previous phrases, these action object are shared and treated as read only auto newAction = Ptr(new Action); newAction->actionType=Action::LeftRecursiveReduce; newAction->actionSource=action->actionSource; newAction->actionTarget=action->actionTarget; newAction->creatorRule=action->creatorRule; newAction->shiftReduceSource=action->shiftReduceSource; newAction->shiftReduceTarget=action->shiftReduceTarget; newAction->creatorRule=shift.key->ownerRule; transition->actions[i]=newAction; } else { errors.Add(Ptr(new ParsingError(state->ownerRule, L"Left recursive reduce action in non-normal-reduce found in rule \"" + state->ownerRule->name + L"\" is not allowed."))); } } } } } } // delete complicated transitions for (auto state : jointPDA->states) { while(true) { bool deleted=false; for (auto t1 : state->transitions) for (auto t2 : state->transitions) if(t1!=t2) { if(Transition::IsEquivalent(t1, t2, true)) { jointPDA->DeleteTransition(t2); deleted=true; goto TRANSITION_DELETED; } } TRANSITION_DELETED: if(!deleted) break; } } } } } } /*********************************************************************** .\PARSINGAUTOMATON_MERGESTATES.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { using namespace collections; using namespace definitions; namespace analyzing { /*********************************************************************** DeleteUnnecessaryStates ***********************************************************************/ void DeleteUnnecessaryStates(Ptr automaton, Ptr ruleInfo, List& newStates) { // delete all states that are not reachable to the end state while(true) { // find a non-end state without out transitions vint deleteCount=0; for(vint i=newStates.Count()-1;i>=0;i--) { State* newState=newStates[i]; if(newState->transitions.Count()==0) { if(newState!=ruleInfo->rootRuleEndState && !newState->endState) { automaton->DeleteState(newState); newStates.RemoveAt(i); } } } if(deleteCount==0) { break; } } } /*********************************************************************** IsMergableCandidate ***********************************************************************/ bool IsMergableCandidate(State* state, Ptr ruleInfo) { if(state==ruleInfo->rootRuleStartState || state==ruleInfo->rootRuleEndState || state==ruleInfo->startState) { return false; } return true; } /*********************************************************************** RearrangeState ***********************************************************************/ std::strong_ordering CompareParsingSymbol(ParsingSymbol* s1, ParsingSymbol* s2) { if (s1 && s2) { std::strong_ordering result = s1->GetType() <=> s2->GetType(); if (result != 0) return result; result = s1->GetName() <=> s2->GetName(); if (result != 0) return result; return result; } else if (s1) { return std::strong_ordering::greater; } else if (s2) { return std::strong_ordering::less; } else { return std::strong_ordering::equal; } } std::strong_ordering CompareTransitionForRearranging(Transition* t1, Transition* t2) { std::strong_ordering result = t1->transitionType <=> t2->transitionType; if (result != 0) return result; result = CompareParsingSymbol(t1->transitionSymbol, t2->transitionSymbol); if (result != 0) return result; return result; } std::strong_ordering CompareActionForRearranging(Ptr a1, Ptr a2) { std::strong_ordering result = a1->actionType <=> a2->actionType; if (result != 0) return result; result = CompareParsingSymbol(a1->actionSource, a2->actionSource); if (result != 0) return result; result = CompareParsingSymbol(a1->actionTarget, a2->actionTarget); if (result != 0) return result; return result; } #undef COMPARE_SYMBOL void RearrangeState(State* state, SortedList& stateContentSorted) { if(!stateContentSorted.Contains(state)) { for (auto transition : state->transitions) { CopyFrom(transition->actions, From(transition->actions).OrderBy(&CompareActionForRearranging)); } CopyFrom(state->transitions, From(state->transitions).OrderBy(&CompareTransitionForRearranging)); stateContentSorted.Add(state); } } /*********************************************************************** MoveActionsForMergingState ***********************************************************************/ void MoveActionsForMergingState(Transition* transition) { // collect all movable actions List> movableActions; for(vint i=transition->actions.Count()-1;i>=0;i--) { switch(transition->actions[i]->actionType) { // Using and Assign actions are not movable case Action::Using: case Action::Assign: break; default: movableActions.Add(transition->actions[i]); transition->actions.RemoveAt(i); } } // copy all movable actions for (auto t : transition->source->inputs) { CopyFrom(t->actions, movableActions, true); } } /*********************************************************************** IsMergableBecause(Transitions|Input) ***********************************************************************/ bool IsMergableBecauseTransitions(State* state1, State* state2) { if(state1->transitions.Count()!=state2->transitions.Count()) return false; if(state1->transitions.Count()==1 && state2->transitions.Count()==1) { Transition* t1=state1->transitions[0]; Transition* t2=state2->transitions[0]; if(CompareTransitionForRearranging(t1, t2)==0 && !Transition::IsEquivalent(t1, t2, false) && t1->target==t2->target) { MoveActionsForMergingState(t1); MoveActionsForMergingState(t2); } } for(vint i=0;itransitions.Count();i++) { Transition* t1=state1->transitions[i]; Transition* t2=state2->transitions[i]; if(!Transition::IsEquivalent(t1, t2, false) || t1->target!=t2->target) { return false; } } return true; } bool IsMergableBecauseInputs(State* state1, State* state2) { if(state1->inputs.Count()!=state2->inputs.Count()) return false; for(vint i=0;iinputs.Count();i++) { Transition* t1=state1->inputs[i]; Transition* t2=state2->inputs[i]; if(!Transition::IsEquivalent(t1, t2, false) || t1->source!=t2->source) { return false; } } return true; } /*********************************************************************** MergeState2ToState1Because(Transitions|Input) ***********************************************************************/ void MergeState2ToState1BecauseTransitions(Ptr automaton, State* state1, State* state2) { // modify state1's expression state1->stateExpression+=L"\r\n"+state2->stateExpression; // retarget state2's input to state1 for(vint i=state2->inputs.Count()-1;i>=0;i--) { Transition* t2=state2->inputs[i]; bool add=true; for (auto t1 : state1->inputs) { if(Transition::IsEquivalent(t1, t2, false) && t1->source==t2->source) { add=false; break; } } if(add) { state1->inputs.Add(t2); t2->target=state1; state2->inputs.RemoveAt(i); } } automaton->DeleteState(state2); } void MergeState2ToState1BecauseInputs(Ptr automaton, State* state1, State* state2) { // modify state1's expression state1->stateExpression+=L"\r\n"+state2->stateExpression; // retarget state2's input to state1 for(vint i=state2->transitions.Count()-1;i>=0;i--) { Transition* t2=state2->transitions[i]; bool add=true; for (auto t1 : state1->transitions) { if(Transition::IsEquivalent(t1, t2, false) && t1->target==t2->target) { add=false; break; } } if(add) { state1->transitions.Add(t2); t2->source=state1; state2->transitions.RemoveAt(i); } } automaton->DeleteState(state2); } /*********************************************************************** MergeStates ***********************************************************************/ void MergeStates(Ptr automaton, Ptr ruleInfo, List& newStates) { SortedList stateContentSorted; while(true) { for(vint i=0;i CreateNondeterministicPDAFromEpsilonPDA(Ptr epsilonPDA) { auto automaton = Ptr(new Automaton(epsilonPDA->symbolManager)); for (auto rule : epsilonPDA->orderedRulesDefs) { // build new rule info auto ruleInfo = epsilonPDA->ruleDefToInfoMap[rule]; auto newRuleInfo = Ptr(new RuleInfo); automaton->AddRuleInfo(rule, newRuleInfo); newRuleInfo->rootRuleStartState=automaton->RootRuleStartState(rule); newRuleInfo->rootRuleEndState=automaton->RootRuleEndState(rule); newRuleInfo->startState=automaton->RuleStartState(rule); // build state mapping and state visiting tracking Dictionary oldNewStateMap; List scanningStates; vint currentStateIndex=0; oldNewStateMap.Add(ruleInfo->rootRuleStartState, newRuleInfo->rootRuleStartState); oldNewStateMap.Add(ruleInfo->rootRuleEndState, newRuleInfo->rootRuleEndState); oldNewStateMap.Add(ruleInfo->startState, newRuleInfo->startState); // begin with a root rule state state scanningStates.Add(ruleInfo->rootRuleStartState); // remove epsilon transitions RemoveEpsilonTransitions(oldNewStateMap, scanningStates, automaton); // stable state orders List newStates; CopyFrom( newStates, From(epsilonPDA->states) .Where([&](auto&& s) {return oldNewStateMap.Keys().Contains(s.Obj()); }) .Select([&](auto&& s) { return oldNewStateMap[s.Obj()]; }) ); DeleteUnnecessaryStates(automaton, newRuleInfo, newStates); MergeStates(automaton, newRuleInfo, newStates); // there should be at east one and only one transition that is TokenBegin from rootRuleStartState // update the startState because the startState may be deleted newRuleInfo->startState=newRuleInfo->rootRuleStartState->transitions[0]->target; // record end states for (auto state : newStates) { if(state->endState) { newRuleInfo->endStates.Add(state); } } } return automaton; } } } } /*********************************************************************** .\PARSINGDEFINITIONS.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { using namespace collections; namespace parsing { namespace definitions { /*********************************************************************** ParsingDefinitionType(Visitor) ***********************************************************************/ void ParsingDefinitionPrimitiveType::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionTokenType::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionSubType::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionArrayType::Accept(IVisitor* visitor) { visitor->Visit(this); } /*********************************************************************** ParsingDefinitionTypeDefinition(Visitor) ***********************************************************************/ void ParsingDefinitionClassMemberDefinition::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionClassDefinition::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionEnumMemberDefinition::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionEnumDefinition::Accept(IVisitor* visitor) { visitor->Visit(this); } /*********************************************************************** ParsingDefinitionGrammar(Visitor) ***********************************************************************/ void ParsingDefinitionPrimitiveGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionTextGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionSequenceGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionAlternativeGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionLoopGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionOptionalGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionCreateGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionAssignGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionUseGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } void ParsingDefinitionSetterGrammar::Accept(IVisitor* visitor) { visitor->Visit(this); } /*********************************************************************** ParsingDefinitionTypeWriter ***********************************************************************/ ParsingDefinitionAttributeWriter::ParsingDefinitionAttributeWriter(const WString& name) { attribute = Ptr(new ParsingDefinitionAttribute); attribute->name=name; } ParsingDefinitionAttributeWriter::ParsingDefinitionAttributeWriter(const ParsingDefinitionAttributeWriter& attributeWriter) { attribute=attributeWriter.attribute; } ParsingDefinitionAttributeWriter& ParsingDefinitionAttributeWriter::Argument(const WString& argument) { attribute->arguments.Add(argument); return *this; } Ptr ParsingDefinitionAttributeWriter::Attribute()const { return attribute; } ParsingDefinitionAttributeWriter Attribute(const WString& name) { return ParsingDefinitionAttributeWriter(name); } /*********************************************************************** ParsingDefinitionTypeWriter ***********************************************************************/ ParsingDefinitionTypeWriter::ParsingDefinitionTypeWriter(Ptr internalType) { type=internalType; } ParsingDefinitionTypeWriter::ParsingDefinitionTypeWriter(const ParsingDefinitionTypeWriter& typeWriter) { type=typeWriter.type; } ParsingDefinitionTypeWriter::ParsingDefinitionTypeWriter(const WString& name) { auto primitiveType = Ptr(new ParsingDefinitionPrimitiveType); primitiveType->name=name; type=primitiveType; } ParsingDefinitionTypeWriter ParsingDefinitionTypeWriter::Sub(const WString& subTypeName)const { auto subType = Ptr(new ParsingDefinitionSubType); subType->parentType=type; subType->subTypeName=subTypeName; return ParsingDefinitionTypeWriter(subType); } ParsingDefinitionTypeWriter ParsingDefinitionTypeWriter::Array()const { auto arrayType = Ptr(new ParsingDefinitionArrayType); arrayType->elementType=type; return ParsingDefinitionTypeWriter(arrayType); } Ptr ParsingDefinitionTypeWriter::Type()const { return type; } ParsingDefinitionTypeWriter Type(const WString& name) { return ParsingDefinitionTypeWriter(name); } ParsingDefinitionTypeWriter TokenType() { auto type = Ptr(new ParsingDefinitionTokenType); return ParsingDefinitionTypeWriter(type); } /*********************************************************************** ParsingDefinitionClassDefinitionWriter ***********************************************************************/ ParsingDefinitionClassDefinitionWriter::ParsingDefinitionClassDefinitionWriter(const WString& name) { definition = Ptr(new ParsingDefinitionClassDefinition); definition->name=name; currentDefinition=definition; } ParsingDefinitionClassDefinitionWriter::ParsingDefinitionClassDefinitionWriter(const WString& name, const ParsingDefinitionTypeWriter& parentType) { definition = Ptr(new ParsingDefinitionClassDefinition); definition->name=name; definition->parentType=parentType.Type(); currentDefinition=definition; } ParsingDefinitionClassDefinitionWriter& ParsingDefinitionClassDefinitionWriter::AmbiguousType(const ParsingDefinitionTypeWriter& ambiguousType) { definition->ambiguousType=ambiguousType.Type(); return *this; } ParsingDefinitionClassDefinitionWriter& ParsingDefinitionClassDefinitionWriter::Member(const WString& name, const ParsingDefinitionTypeWriter& type, const WString& unescapingFunction) { auto member = Ptr(new ParsingDefinitionClassMemberDefinition); member->name=name; member->type=type.Type(); member->unescapingFunction=unescapingFunction; definition->members.Add(member); currentDefinition=member; return *this; } ParsingDefinitionClassDefinitionWriter& ParsingDefinitionClassDefinitionWriter::SubType(const ParsingDefinitionTypeDefinitionWriter& type) { definition->subTypes.Add(type.Definition()); return *this; } ParsingDefinitionClassDefinitionWriter& ParsingDefinitionClassDefinitionWriter::Attribute(const ParsingDefinitionAttributeWriter& attribute) { currentDefinition->attributes.Add(attribute.Attribute()); return *this; } Ptr ParsingDefinitionClassDefinitionWriter::Definition()const { return definition; } ParsingDefinitionClassDefinitionWriter Class(const WString& name) { return ParsingDefinitionClassDefinitionWriter(name); } ParsingDefinitionClassDefinitionWriter Class(const WString& name, const ParsingDefinitionTypeWriter& parentType) { return ParsingDefinitionClassDefinitionWriter(name, parentType); } /*********************************************************************** ParsingDefinitionEnumDefinitionWriter ***********************************************************************/ ParsingDefinitionEnumDefinitionWriter::ParsingDefinitionEnumDefinitionWriter(const WString& name) { definition = Ptr(new ParsingDefinitionEnumDefinition); definition->name=name; currentDefinition=definition; } ParsingDefinitionEnumDefinitionWriter& ParsingDefinitionEnumDefinitionWriter::Member(const WString& name) { auto member = Ptr(new ParsingDefinitionEnumMemberDefinition); member->name=name; definition->members.Add(member); currentDefinition=member; return *this; } ParsingDefinitionEnumDefinitionWriter& ParsingDefinitionEnumDefinitionWriter::Attribute(const ParsingDefinitionAttributeWriter& attribute) { currentDefinition->attributes.Add(attribute.Attribute()); return *this; } Ptr ParsingDefinitionEnumDefinitionWriter::Definition()const { return definition; } ParsingDefinitionEnumDefinitionWriter Enum(const WString& name) { return ParsingDefinitionEnumDefinitionWriter(name); } /*********************************************************************** ParsingDefinitionGrammarWriter ***********************************************************************/ ParsingDefinitionGrammarWriter::ParsingDefinitionGrammarWriter(Ptr internalGrammar) { grammar=internalGrammar; } ParsingDefinitionGrammarWriter::ParsingDefinitionGrammarWriter(const ParsingDefinitionGrammarWriter& grammarWriter) { grammar=grammarWriter.grammar; } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::operator+(const ParsingDefinitionGrammarWriter& next)const { auto sequence = Ptr(new ParsingDefinitionSequenceGrammar); sequence->first=grammar; sequence->second=next.Grammar(); return ParsingDefinitionGrammarWriter(sequence); } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::operator|(const ParsingDefinitionGrammarWriter& next)const { auto alternative = Ptr(new ParsingDefinitionAlternativeGrammar); alternative->first=grammar; alternative->second=next.Grammar(); return ParsingDefinitionGrammarWriter(alternative); } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::operator*()const { auto loop = Ptr(new ParsingDefinitionLoopGrammar); loop->grammar=grammar; return ParsingDefinitionGrammarWriter(loop); } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::As(const ParsingDefinitionTypeWriter& type)const { auto create = Ptr(new ParsingDefinitionCreateGrammar); create->grammar=grammar; create->type=type.Type(); return ParsingDefinitionGrammarWriter(create); } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::operator[](const WString& memberName)const { auto assign = Ptr(new ParsingDefinitionAssignGrammar); assign->grammar=grammar; assign->memberName=memberName; return ParsingDefinitionGrammarWriter(assign); } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::operator!()const { auto use = Ptr(new ParsingDefinitionUseGrammar); use->grammar=grammar; return ParsingDefinitionGrammarWriter(use); } ParsingDefinitionGrammarWriter ParsingDefinitionGrammarWriter::Set(const WString& memberName, const WString& value)const { auto setter = Ptr(new ParsingDefinitionSetterGrammar); setter->grammar=grammar; setter->memberName=memberName; setter->value=value; return ParsingDefinitionGrammarWriter(setter); } Ptr ParsingDefinitionGrammarWriter::Grammar()const { return grammar; } ParsingDefinitionGrammarWriter Rule(const WString& name) { auto grammar = Ptr(new ParsingDefinitionPrimitiveGrammar); grammar->name=name; return ParsingDefinitionGrammarWriter(grammar); } ParsingDefinitionGrammarWriter Text(const WString& text) { auto grammar = Ptr(new ParsingDefinitionTextGrammar); grammar->text=text; return ParsingDefinitionGrammarWriter(grammar); } ParsingDefinitionGrammarWriter Opt(const ParsingDefinitionGrammarWriter& writer) { auto grammar = Ptr(new ParsingDefinitionOptionalGrammar); grammar->grammar=writer.Grammar(); return ParsingDefinitionGrammarWriter(grammar); } /*********************************************************************** ParsingDefinitionTokenDefinitionWriter ***********************************************************************/ ParsingDefinitionTokenDefinitionWriter::ParsingDefinitionTokenDefinitionWriter(ParsingDefinitionWriter& _owner, Ptr _token) :owner(_owner) ,token(_token) { } ParsingDefinitionTokenDefinitionWriter& ParsingDefinitionTokenDefinitionWriter::Attribute(const ParsingDefinitionAttributeWriter& attribute) { token->attributes.Add(attribute.Attribute()); return *this; } ParsingDefinitionWriter& ParsingDefinitionTokenDefinitionWriter::EndToken() { return owner; } /*********************************************************************** ParsingDefinitionRuleDefinitionWriter ***********************************************************************/ ParsingDefinitionRuleDefinitionWriter::ParsingDefinitionRuleDefinitionWriter(ParsingDefinitionWriter& _owner, Ptr _rule) :owner(_owner) ,rule(_rule) { } ParsingDefinitionRuleDefinitionWriter& ParsingDefinitionRuleDefinitionWriter::Imply(const ParsingDefinitionGrammarWriter& grammar) { rule->grammars.Add(grammar.Grammar()); return *this; } ParsingDefinitionRuleDefinitionWriter& ParsingDefinitionRuleDefinitionWriter::Attribute(const ParsingDefinitionAttributeWriter& attribute) { rule->attributes.Add(attribute.Attribute()); return *this; } ParsingDefinitionWriter& ParsingDefinitionRuleDefinitionWriter::EndRule() { return owner; } /*********************************************************************** ParsingDefinitionWriter ***********************************************************************/ ParsingDefinitionWriter::ParsingDefinitionWriter() { definition = Ptr(new ParsingDefinition); } ParsingDefinitionWriter& ParsingDefinitionWriter::Type(const ParsingDefinitionTypeDefinitionWriter& type) { definition->types.Add(type.Definition()); return *this; } ParsingDefinitionWriter& ParsingDefinitionWriter::Token(const WString& name, const WString& regex) { return TokenAtt(name, regex).EndToken(); } ParsingDefinitionTokenDefinitionWriter ParsingDefinitionWriter::TokenAtt(const WString& name, const WString& regex) { auto token = Ptr(new ParsingDefinitionTokenDefinition); token->name=name; token->regex=regex; token->discard=false; definition->tokens.Add(token); return ParsingDefinitionTokenDefinitionWriter(*this, token); } ParsingDefinitionWriter& ParsingDefinitionWriter::Discard(const WString& name, const WString& regex) { auto token = Ptr(new ParsingDefinitionTokenDefinition); token->name=name; token->regex=regex; token->discard=true; definition->tokens.Add(token); return *this; } ParsingDefinitionRuleDefinitionWriter ParsingDefinitionWriter::Rule(const WString& name, const ParsingDefinitionTypeWriter& type) { auto rule = Ptr(new ParsingDefinitionRuleDefinition); rule->name=name; rule->type=type.Type(); definition->rules.Add(rule); return ParsingDefinitionRuleDefinitionWriter(*this, rule); } Ptr ParsingDefinitionWriter::Definition()const { return definition; } } } } /*********************************************************************** .\PARSINGDEFINITIONS_CREATEPARSERDEFINITION.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { namespace definitions { using namespace collections; /*********************************************************************** Bootstrap ***********************************************************************/ Ptr CreateParserDefinition() { ParsingDefinitionWriter definitionWriter; definitionWriter .Type( Class(L"AttributeDef") .Member(L"name", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Attribute")) .Member(L"arguments", TokenType().Array()) ) .Type( Class(L"DefBase") .Member(L"attributes", Type(L"AttributeDef").Array()) ) //------------------------------------- .Type( Class(L"TypeObj") ) .Type( Class(L"PrimitiveTypeObj", Type(L"TypeObj")) .Member(L"name", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Type")) ) .Type( Class(L"TokenTypeObj", Type(L"TypeObj")) ) .Type( Class(L"SubTypeObj", Type(L"TypeObj")) .Member(L"parentType", Type(L"TypeObj")) .Member(L"name", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Type")) ) .Type( Class(L"ArrayTypeObj", Type(L"TypeObj")) .Member(L"elementType", Type(L"TypeObj")) ) //------------------------------------- .Type( Class(L"TypeDef", Type(L"DefBase")) .Member(L"name", TokenType()) .Attribute(Attribute(L"Color").Argument(L"Type")) ) .Type( Class(L"ClassMemberDef", Type(L"DefBase")) .Member(L"type", Type(L"TypeObj")) .Member(L"name", TokenType()) .Member(L"unescapingFunction", TokenType()) ) .Type( Class(L"ClassTypeDef", Type(L"TypeDef")) .Member(L"ambiguousType", Type(L"TypeObj")) .Member(L"parentType", Type(L"TypeObj")) .Member(L"members", Type(L"ClassMemberDef").Array()) .Member(L"subTypes", Type(L"TypeDef").Array()) ) .Type( Class(L"EnumMemberDef", Type(L"DefBase")) .Member(L"name", TokenType()) ) .Type( Class(L"EnumTypeDef", Type(L"TypeDef")) .Member(L"members", Type(L"EnumMemberDef").Array()) ) //------------------------------------- .Type( Class(L"GrammarDef") ) .Type( Class(L"PrimitiveGrammarDef", Type(L"GrammarDef")) .Member(L"name", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Token").Argument(L"Rule")) ) .Type( Class(L"TextGrammarDef", Type(L"GrammarDef")) .Member(L"text", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Literal")) ) .Type( Class(L"SequenceGrammarDef", Type(L"GrammarDef")) .Member(L"first", Type(L"GrammarDef")) .Member(L"second", Type(L"GrammarDef")) ) .Type( Class(L"AlternativeGrammarDef", Type(L"GrammarDef")) .Member(L"first", Type(L"GrammarDef")) .Member(L"second", Type(L"GrammarDef")) ) .Type( Class(L"LoopGrammarDef", Type(L"GrammarDef")) .Member(L"grammar", Type(L"GrammarDef")) ) .Type( Class(L"OptionalGrammarDef", Type(L"GrammarDef")) .Member(L"grammar", Type(L"GrammarDef")) ) .Type( Class(L"CreateGrammarDef", Type(L"GrammarDef")) .Member(L"grammar", Type(L"GrammarDef")) .Member(L"type", Type(L"TypeObj")) ) .Type( Class(L"AssignGrammarDef", Type(L"GrammarDef")) .Member(L"grammar", Type(L"GrammarDef")) .Member(L"memberName", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Field")) ) .Type( Class(L"UseGrammarDef", Type(L"GrammarDef")) .Member(L"grammar", Type(L"GrammarDef")) ) .Type( Class(L"SetterGrammarDef", Type(L"GrammarDef")) .Member(L"grammar", Type(L"GrammarDef")) .Member(L"memberName", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"Field")) .Member(L"value", TokenType()) .Attribute(Attribute(L"Semantic").Argument(L"EnumValue")) ) //------------------------------------- .Type( Class(L"TokenDef", Type(L"DefBase")) .SubType( Enum(L"DiscardOption") .Member(L"DiscardToken") .Member(L"KeepToken") ) .Member(L"name", TokenType()) .Attribute(Attribute(L"Color").Argument(L"Token")) .Member(L"regex", TokenType()) .Member(L"discard", Type(L"DiscardOption")) ) .Type( Class(L"RuleDef", Type(L"DefBase")) .Member(L"name", TokenType()) .Attribute(Attribute(L"Color").Argument(L"Rule")) .Member(L"type", Type(L"TypeObj")) .Member(L"grammars", Type(L"GrammarDef").Array()) ) .Type( Class(L"ParserDef") .Member(L"definitions", Type(L"DefBase").Array()) ) //------------------------------------- .TokenAtt(L"CLASS", L"class") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"AMBIGUOUS", L"ambiguous") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"ENUM", L"enum") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"TOKEN", L"token") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"DISCARDTOKEN", L"discardtoken") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"RULE", L"rule") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"AS", L"as") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .TokenAtt(L"WITH", L"with") .Attribute(Attribute(L"Color").Argument(L"Keyword")) .Attribute(Attribute(L"Candidate")) .EndToken() .Token(L"OPEN", L"/{") .Token(L"CLOSE", L"/}") .Token(L"SEMICOLON", L";") .Token(L"COLON", L":") .Token(L"COMMA", L",") .Token(L"DOT", L".") .Token(L"ASSIGN", L"/=") .Token(L"USING", L"/!") .Token(L"OR", L"/|") .Token(L"OPTOPEN", L"/[") .Token(L"OPTCLOSE", L"/]") .Token(L"PREOPEN", L"/(") .Token(L"PRECLOSE", L"/)") .TokenAtt(L"ATT", L"@") .Attribute(Attribute(L"Color").Argument(L"Attribute")) .EndToken() .TokenAtt(L"NAME", L"[a-zA-Z_]/w*") .Attribute(Attribute(L"Color").Argument(L"Default")) .Attribute(Attribute(L"ContextColor")) .Attribute(Attribute(L"AutoComplete")) .EndToken() .TokenAtt(L"STRING", L"\"([^\"]|\"\")*\"") .Attribute(Attribute(L"Color").Argument(L"String")) .Attribute(Attribute(L"AutoComplete")) .EndToken() .Discard(L"SPACE", L"/s+") .Discard(L"COMMENT", L"////[^\\r\\n]*") //------------------------------------- .Rule(L"Attribute", Type(L"AttributeDef")) .Imply( (Text(L"@") + Rule(L"NAME")[L"name"] + Text(L"(") + Opt(Rule(L"STRING")[L"arguments"] + *(Text(L",") + Rule(L"STRING")[L"arguments"])) + Text(L")")) .As(Type(L"AttributeDef")) ) .EndRule() //------------------------------------- .Rule(L"Type", Type(L"TypeObj")) .Imply( (Rule(L"NAME")[L"name"]) .As(Type(L"PrimitiveTypeObj")) ) .Imply( Text(L"token") .As(Type(L"TokenTypeObj")) ) .Imply( (Rule(L"Type")[L"parentType"] + Text(L".") + Rule(L"NAME")[L"name"]) .As(Type(L"SubTypeObj")) ) .Imply( (Rule(L"Type")[L"elementType"] + Text(L"[") + Text(L"]")) .As(Type(L"ArrayTypeObj")) ) .EndRule() //------------------------------------- .Rule(L"EnumMember", Type(L"EnumMemberDef")) .Imply( ( Rule(L"NAME")[L"name"] + Opt(Rule(L"Attribute")[L"attributes"] + *(Text(L",") + Rule(L"Attribute")[L"attributes"])) + Text(L",") ) .As(Type(L"EnumMemberDef")) ) .EndRule() .Rule(L"Enum", Type(L"EnumTypeDef")) .Imply( ( Text(L"enum") + Rule(L"NAME")[L"name"] + Opt(Rule(L"Attribute")[L"attributes"] + *(Text(L",") + Rule(L"Attribute")[L"attributes"])) + Text(L"{") + *(Rule(L"EnumMember")[L"members"]) + Text(L"}") ) .As(Type(L"EnumTypeDef")) ) .EndRule() .Rule(L"ClassMember", Type(L"ClassMemberDef")) .Imply( ( Rule(L"Type")[L"type"] + Rule(L"NAME")[L"name"] + Opt(Text(L"(") + Rule(L"NAME")[L"unescapingFunction"] + Text(L")")) + Opt(Rule(L"Attribute")[L"attributes"] + *(Text(L",") + Rule(L"Attribute")[L"attributes"])) + Text(L";") ) .As(Type(L"ClassMemberDef")) ) .EndRule() .Rule(L"Class", Type(L"ClassTypeDef")) .Imply( ( Text(L"class") + Rule(L"NAME")[L"name"] + Opt(Text(L"ambiguous") + Text(L"(") + Rule(L"Type")[L"ambiguousType"] + Text(L")")) + Opt(Text(L":") + Rule(L"Type")[L"parentType"]) + Opt(Rule(L"Attribute")[L"attributes"] + *(Text(L",") + Rule(L"Attribute")[L"attributes"])) + Text(L"{") + *(Rule(L"ClassMember")[L"members"] | Rule(L"TypeDecl")[L"subTypes"]) + Text(L"}") ) .As(Type(L"ClassTypeDef")) ) .EndRule() .Rule(L"TypeDecl", Type(L"TypeDef")) .Imply(!Rule(L"Enum") | !Rule(L"Class")) .EndRule() //------------------------------------ .Rule(L"PrimitiveGrammar", Type(L"GrammarDef")) .Imply( (Rule(L"NAME")[L"name"]) .As(Type(L"PrimitiveGrammarDef")) ) .Imply( (Rule(L"STRING")[L"text"]) .As(Type(L"TextGrammarDef")) ) .Imply( (Rule(L"PrimitiveGrammar")[L"grammar"] + Text(L":") + Rule(L"NAME")[L"memberName"]) .As(Type(L"AssignGrammarDef")) ) .Imply( (Text(L"!") + Rule(L"PrimitiveGrammar")[L"grammar"]) .As(Type(L"UseGrammarDef")) ) .Imply( (Text(L"[") + Rule(L"Grammar")[L"grammar"] + Text(L"]")) .As(Type(L"OptionalGrammarDef")) ) .Imply( (Text(L"{") + Rule(L"Grammar")[L"grammar"] + Text(L"}")) .As(Type(L"LoopGrammarDef")) ) .Imply( (Text(L"(") + !Rule(L"Grammar") + Text(L")")) ) .EndRule() .Rule(L"SequenceGrammar", Type(L"GrammarDef")) .Imply( !Rule(L"PrimitiveGrammar") ) .Imply( (Rule(L"SequenceGrammar")[L"first"] + Rule(L"PrimitiveGrammar")[L"second"]) .As(Type(L"SequenceGrammarDef")) ) .EndRule() .Rule(L"AlternativeGrammar", Type(L"GrammarDef")) .Imply( !Rule(L"SequenceGrammar") ) .Imply( (Rule(L"AlternativeGrammar")[L"first"] + Text(L"|") + Rule(L"SequenceGrammar")[L"second"]) .As(Type(L"AlternativeGrammarDef")) ) .EndRule() .Rule(L"Grammar", Type(L"GrammarDef")) .Imply( !Rule(L"AlternativeGrammar") ) .Imply( (Rule(L"Grammar")[L"grammar"] + Text(L"as") + Rule(L"Type")[L"type"]) .As(Type(L"CreateGrammarDef")) ) .Imply( (Rule(L"Grammar")[L"grammar"] + Text(L"with") + Text(L"{") + Rule(L"NAME")[L"memberName"] + Text(L"=") + Rule(L"STRING")[L"value"] + Text(L"}")) .As(Type(L"SetterGrammarDef")) ) .EndRule() //------------------------------------ .Rule(L"TokenDecl", Type(L"TokenDef")) .Imply( ( Text(L"token") + Rule(L"NAME")[L"name"] + Text(L"=") + Rule(L"STRING")[L"regex"] + Opt(Rule(L"Attribute")[L"attributes"] + *(Text(L",") + Rule(L"Attribute")[L"attributes"])) + Text(L";") ) .As(Type(L"TokenDef")) .Set(L"discard", L"KeepToken") ) .Imply( (Text(L"discardtoken") + Rule(L"NAME")[L"name"] + Text(L"=") + Rule(L"STRING")[L"regex"] + Text(L";")) .As(Type(L"TokenDef")) .Set(L"discard", L"DiscardToken") ) .EndRule() .Rule(L"RuleDecl", Type(L"RuleDef")) .Imply( ( Text(L"rule") + Rule(L"Type")[L"type"] + Rule(L"NAME")[L"name"] + Opt(Rule(L"Attribute")[L"attributes"] + *(Text(L",") + Rule(L"Attribute")[L"attributes"])) + *(Text(L"=") + Rule(L"Grammar")[L"grammars"]) + Text(L";") ) .As(Type(L"RuleDef")) ) .EndRule() //------------------------------------ .Rule(L"ParserDecl", Type(L"ParserDef")) .Imply( ( *( Rule(L"TypeDecl")[L"definitions"] | Rule(L"TokenDecl")[L"definitions"] | Rule(L"RuleDecl")[L"definitions"] ) +( Rule(L"TypeDecl")[L"definitions"] | Rule(L"TokenDecl")[L"definitions"] | Rule(L"RuleDecl")[L"definitions"] ) ) .As(Type(L"ParserDef")) ) .EndRule() ; return definitionWriter.Definition(); } WString DeserializeString(const WString& value) { if(value.Length()>=2 && value[0]==L'"' && value[value.Length()-1]==L'"') { Array chars(value.Length()); memset(&chars[0], 0, chars.Count()*sizeof(wchar_t)); const wchar_t* reading=value.Buffer()+1; wchar_t* writing=&chars[0]; while(*reading) { if(*reading!=L'"') { *writing++=*reading++; } else if(reading[1]!=L'"') { break; } else { *writing++=L'"'; reading+=2; } } return &chars[0]; } return L""; } WString DeserializeString(Ptr token) { const WString& value=token->GetValue(); return DeserializeString(value); } void SetName(WString& target, Ptr node) { Ptr token=node.Cast(); if(token) { target=token->GetValue(); } } void SetText(WString& target, Ptr node) { Ptr token=node.Cast(); if(token) { target=DeserializeString(token); } } extern Ptr Deserialize(Ptr node); template void SetArray(List>& target, Ptr node) { Ptr source=node.Cast(); if(source) { for(vint i=0;iCount();i++) { target.Add(Deserialize(source->GetItem(i).Cast()).Cast()); } } } void SetArray(List& target, Ptr node) { Ptr source=node.Cast(); if(source) { for(vint i=0;iCount();i++) { WString name; SetName(name, source->GetItem(i)); target.Add(name); } } } template void SetMember(Ptr& target, Ptr node) { Ptr source=node.Cast(); if(source) { target=Deserialize(source).Cast(); } } Ptr Deserialize(Ptr node) { if(!node) { return 0; } else if(node->GetType()==L"AttributeDef") { auto target = Ptr(new ParsingDefinitionAttribute); SetName(target->name, node->GetMember(L"name")); SetArray(target->arguments, node->GetMember(L"arguments")); for(vint i=0;iarguments.Count();i++) { target->arguments[i]=DeserializeString(target->arguments[i]); } return target; } else if(node->GetType()==L"PrimitiveTypeObj") { auto target = Ptr(new ParsingDefinitionPrimitiveType); SetName(target->name, node->GetMember(L"name")); return target; } else if(node->GetType()==L"TokenTypeObj") { auto target = Ptr(new ParsingDefinitionTokenType); return target; } else if(node->GetType()==L"SubTypeObj") { auto target = Ptr(new ParsingDefinitionSubType); SetMember(target->parentType, node->GetMember(L"parentType")); SetName(target->subTypeName, node->GetMember(L"name")); return target; } else if(node->GetType()==L"ArrayTypeObj") { auto target = Ptr(new ParsingDefinitionArrayType); SetMember(target->elementType, node->GetMember(L"elementType")); return target; } else if(node->GetType()==L"ClassMemberDef") { auto target = Ptr(new ParsingDefinitionClassMemberDefinition); SetArray(target->attributes, node->GetMember(L"attributes")); SetMember(target->type, node->GetMember(L"type")); SetName(target->name, node->GetMember(L"name")); SetName(target->unescapingFunction, node->GetMember(L"unescapingFunction")); return target; } else if(node->GetType()==L"ClassTypeDef") { auto target = Ptr(new ParsingDefinitionClassDefinition); SetArray(target->attributes, node->GetMember(L"attributes")); SetMember(target->ambiguousType, node->GetMember(L"ambiguousType")); SetMember(target->parentType, node->GetMember(L"parentType")); SetName(target->name, node->GetMember(L"name")); SetArray(target->members, node->GetMember(L"members")); SetArray(target->subTypes, node->GetMember(L"subTypes")); return target; } else if(node->GetType()==L"EnumMemberDef") { auto target = Ptr(new ParsingDefinitionEnumMemberDefinition); SetArray(target->attributes, node->GetMember(L"attributes")); SetName(target->name, node->GetMember(L"name")); return target; } else if(node->GetType()==L"EnumTypeDef") { auto target = Ptr(new ParsingDefinitionEnumDefinition); SetArray(target->attributes, node->GetMember(L"attributes")); SetName(target->name, node->GetMember(L"name")); SetArray(target->members, node->GetMember(L"members")); return target; } else if(node->GetType()==L"PrimitiveGrammarDef") { auto target = Ptr(new ParsingDefinitionPrimitiveGrammar); SetName(target->name, node->GetMember(L"name")); return target; } else if(node->GetType()==L"TextGrammarDef") { auto target = Ptr(new ParsingDefinitionTextGrammar); SetText(target->text, node->GetMember(L"text")); return target; } else if(node->GetType()==L"SequenceGrammarDef") { auto target = Ptr(new ParsingDefinitionSequenceGrammar); SetMember(target->first, node->GetMember(L"first")); SetMember(target->second, node->GetMember(L"second")); return target; } else if(node->GetType()==L"AlternativeGrammarDef") { auto target = Ptr(new ParsingDefinitionAlternativeGrammar); SetMember(target->first, node->GetMember(L"first")); SetMember(target->second, node->GetMember(L"second")); return target; } else if(node->GetType()==L"LoopGrammarDef") { auto target = Ptr(new ParsingDefinitionLoopGrammar); SetMember(target->grammar, node->GetMember(L"grammar")); return target; } else if(node->GetType()==L"OptionalGrammarDef") { auto target = Ptr(new ParsingDefinitionOptionalGrammar); SetMember(target->grammar, node->GetMember(L"grammar")); return target; } else if(node->GetType()==L"CreateGrammarDef") { auto target = Ptr(new ParsingDefinitionCreateGrammar); SetMember(target->grammar, node->GetMember(L"grammar")); SetMember(target->type, node->GetMember(L"type")); return target; } else if(node->GetType()==L"AssignGrammarDef") { auto target = Ptr(new ParsingDefinitionAssignGrammar); SetMember(target->grammar, node->GetMember(L"grammar")); SetName(target->memberName, node->GetMember(L"memberName")); return target; } else if(node->GetType()==L"UseGrammarDef") { auto target = Ptr(new ParsingDefinitionUseGrammar); SetMember(target->grammar, node->GetMember(L"grammar")); return target; } else if(node->GetType()==L"SetterGrammarDef") { auto target = Ptr(new ParsingDefinitionSetterGrammar); SetMember(target->grammar, node->GetMember(L"grammar")); SetName(target->memberName, node->GetMember(L"memberName")); SetText(target->value, node->GetMember(L"value")); return target; } else if(node->GetType()==L"TokenDef") { auto target = Ptr(new ParsingDefinitionTokenDefinition); SetArray(target->attributes, node->GetMember(L"attributes")); SetName(target->name, node->GetMember(L"name")); SetText(target->regex, node->GetMember(L"regex")); Ptr token=node->GetMember(L"discard").Cast(); target->discard=(token && token->GetValue()==L"DiscardToken"); return target; } else if(node->GetType()==L"RuleDef") { auto target = Ptr(new ParsingDefinitionRuleDefinition); SetArray(target->attributes, node->GetMember(L"attributes")); SetName(target->name, node->GetMember(L"name")); SetMember(target->type, node->GetMember(L"type")); SetArray(target->grammars, node->GetMember(L"grammars")); return target; } else if(node->GetType()==L"ParserDef") { auto target = Ptr(new ParsingDefinition); Ptr defs=node->GetMember(L"definitions").Cast(); if(defs) { vint count=defs->Count(); for(vint i=0;i def=defs->GetItem(i).Cast(); Ptr defObject=Deserialize(def); if(Ptr defType=defObject.Cast()) { target->types.Add(defType); } else if(Ptr defToken=defObject.Cast()) { target->tokens.Add(defToken); } else if(Ptr defRule=defObject.Cast()) { target->rules.Add(defRule); } } } return target; } else { return 0; } } Ptr DeserializeDefinition(Ptr node) { return Deserialize(node.Cast()).Cast(); } } } } /*********************************************************************** .\PARSINGLOGGING.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { using namespace stream; using namespace collections; namespace parsing { namespace definitions { void LogString(const WString& input, TextWriter& writer) { writer.WriteChar(L'\"'); for(int i=0;iattributes.Count();i++) { ParsingDefinitionAttribute* att=definition->attributes[i].Obj(); if(i>0) writer.WriteChar(L','); writer.WriteString(L" @"); writer.WriteString(att->name); writer.WriteChar(L'('); for(vint j=0;jarguments.Count();j++) { if(j>0) writer.WriteString(L", "); LogString(att->arguments[j], writer); } writer.WriteChar(L')'); } } /*********************************************************************** Logger (ParsingDefinitionType) ***********************************************************************/ class ParsingDefinitionTypeLogger : public Object, public ParsingDefinitionType::IVisitor { public: TextWriter& writer; ParsingDefinitionTypeLogger(TextWriter& _writer) :writer(_writer) { } static void LogInternal(ParsingDefinitionType* type, TextWriter& writer) { ParsingDefinitionTypeLogger visitor(writer); type->Accept(&visitor); } void Visit(ParsingDefinitionPrimitiveType* node)override { writer.WriteString(node->name); } void Visit(ParsingDefinitionTokenType* node)override { writer.WriteString(L"token"); } void Visit(ParsingDefinitionSubType* node)override { LogInternal(node->parentType.Obj(), writer); writer.WriteString(L"."); writer.WriteString(node->subTypeName); } void Visit(ParsingDefinitionArrayType* node)override { LogInternal(node->elementType.Obj(), writer); writer.WriteString(L"[]"); } }; void Log(ParsingDefinitionType* type, TextWriter& writer) { ParsingDefinitionTypeLogger::LogInternal(type, writer); } /*********************************************************************** Logger (ParsingDefinitionTypeDefinition) ***********************************************************************/ class ParsingDefinitionTypeDefinitionLogger : public Object, public ParsingDefinitionTypeDefinition::IVisitor { public: WString prefix; TextWriter& writer; static void LogInternal(ParsingDefinitionTypeDefinition* definition, const WString& prefix, TextWriter& writer) { ParsingDefinitionTypeDefinitionLogger visitor(prefix, writer); definition->Accept(&visitor); } ParsingDefinitionTypeDefinitionLogger(const WString& _prefix, TextWriter& _writer) :prefix(_prefix) ,writer(_writer) { } void Visit(ParsingDefinitionClassMemberDefinition* node)override { writer.WriteString(prefix); Log(node->type.Obj(), writer); writer.WriteString(L" "); writer.WriteString(node->name); if(node->unescapingFunction!=L"") { writer.WriteString(L" ("); writer.WriteString(node->unescapingFunction); writer.WriteString(L")"); } LogAttributeList(node, writer); writer.WriteLine(L";"); } void Visit(ParsingDefinitionClassDefinition* node)override { writer.WriteString(prefix); writer.WriteString(L"class "); writer.WriteString(node->name); if(node->ambiguousType) { writer.WriteString(L" ambiguous("); Log(node->ambiguousType.Obj(), writer); writer.WriteString(L")"); } if(node->parentType) { writer.WriteString(L" : "); Log(node->parentType.Obj(), writer); } LogAttributeList(node, writer); writer.WriteLine(L""); writer.WriteString(prefix); writer.WriteLine(L"{"); for(int i=0;isubTypes.Count();i++) { LogInternal(node->subTypes[i].Obj(), prefix+L" ", writer); writer.WriteLine(L""); } for(int i=0;imembers.Count();i++) { LogInternal(node->members[i].Obj(), prefix+L" ", writer); } writer.WriteString(prefix); writer.WriteLine(L"}"); } void Visit(ParsingDefinitionEnumMemberDefinition* node)override { writer.WriteString(prefix); writer.WriteString(node->name); LogAttributeList(node, writer); writer.WriteLine(L","); } void Visit(ParsingDefinitionEnumDefinition* node)override { writer.WriteString(prefix); writer.WriteString(L"enum "); writer.WriteString(node->name); LogAttributeList(node, writer); writer.WriteLine(L""); writer.WriteString(prefix); writer.WriteLine(L"{"); for(int i=0;imembers.Count();i++) { LogInternal(node->members[i].Obj(), prefix+L" ", writer); } writer.WriteString(prefix); writer.WriteLine(L"}"); } }; void Log(ParsingDefinitionTypeDefinition* definition, const WString& prefix, TextWriter& writer) { ParsingDefinitionTypeDefinitionLogger::LogInternal(definition, prefix, writer); } /*********************************************************************** Logger (ParsingDefinitionGrammar) ***********************************************************************/ #define PRIORITY_NONE 0 #define PRIORITY_CREATE 1 #define PRIORITY_SET 1 #define PRIORITY_ALTERNATIVE 2 #define PRIORITY_SEQUENCE 3 #define PRIORITY_USE 4 #define PRIORITY_ASSIGN 4 class ParsingDefinitionGrammarLogger : public Object, public ParsingDefinitionGrammar::IVisitor { public: TextWriter& writer; int parentPriority; ParsingDefinitionGrammar* stateNode; bool beforeNode; ParsingDefinitionGrammarLogger(TextWriter& _writer, int _parentPriority, ParsingDefinitionGrammar* _stateNode, bool _beforeNode) :writer(_writer) ,parentPriority(_parentPriority) ,stateNode(_stateNode) ,beforeNode(_beforeNode) { } static void LogInternal(ParsingDefinitionGrammar* grammar, int parentPriority, ParsingDefinitionGrammar* stateNode, bool beforeNode, TextWriter& writer) { if(grammar==stateNode && beforeNode) { writer.WriteString(L"@"); } ParsingDefinitionGrammarLogger visitor(writer, parentPriority, stateNode, beforeNode); grammar->Accept(&visitor); if(grammar==stateNode && !beforeNode) { writer.WriteString(L"@"); } } void LogInternal(ParsingDefinitionGrammar* grammar, int parentPriority, TextWriter& writer) { LogInternal(grammar, parentPriority, stateNode, beforeNode, writer); } void Visit(ParsingDefinitionPrimitiveGrammar* node)override { writer.WriteString(node->name); } void Visit(ParsingDefinitionTextGrammar* node)override { LogString(node->text, writer); } void Visit(ParsingDefinitionSequenceGrammar* node)override { int priority=PRIORITY_SEQUENCE; if(parentPriority>priority) { writer.WriteString(L"( "); } LogInternal(node->first.Obj(), priority, writer); writer.WriteString(L" "); LogInternal(node->second.Obj(), priority, writer); if(parentPriority>priority) { writer.WriteString(L" )"); } } void Visit(ParsingDefinitionAlternativeGrammar* node)override { int priority=PRIORITY_ALTERNATIVE; if(parentPriority>priority) { writer.WriteString(L"( "); } LogInternal(node->first.Obj(), priority, writer); writer.WriteString(L" | "); LogInternal(node->second.Obj(), priority, writer); if(parentPriority>priority) { writer.WriteString(L" )"); } } void Visit(ParsingDefinitionLoopGrammar* node)override { writer.WriteString(L"{ "); LogInternal(node->grammar.Obj(), PRIORITY_NONE, writer); writer.WriteString(L" }"); } void Visit(ParsingDefinitionOptionalGrammar* node)override { writer.WriteString(L"[ "); LogInternal(node->grammar.Obj(), PRIORITY_NONE, writer); writer.WriteString(L" ]"); } void Visit(ParsingDefinitionCreateGrammar* node)override { int priority=PRIORITY_CREATE; if(parentPriority>priority) { writer.WriteString(L"( "); } LogInternal(node->grammar.Obj(), priority, writer); writer.WriteString(L" as "); Log(node->type.Obj(), writer); if(parentPriority>priority) { writer.WriteString(L" )"); } } void Visit(ParsingDefinitionAssignGrammar* node)override { int priority=PRIORITY_ASSIGN; if(parentPriority>priority) { writer.WriteString(L"( "); } LogInternal(node->grammar.Obj(), priority, writer); writer.WriteString(L" : "); writer.WriteString(node->memberName); if(parentPriority>priority) { writer.WriteString(L" )"); } } void Visit(ParsingDefinitionUseGrammar* node)override { int priority=PRIORITY_USE; if(parentPriority>priority) { writer.WriteString(L"( "); } writer.WriteString(L"!"); LogInternal(node->grammar.Obj(), priority, writer); if(parentPriority>priority) { writer.WriteString(L" )"); } } void Visit(ParsingDefinitionSetterGrammar* node)override { int priority=PRIORITY_SET; if(parentPriority>priority) { writer.WriteString(L"( "); } LogInternal(node->grammar.Obj(), priority, writer); writer.WriteString(L" with { "); writer.WriteString(node->memberName); writer.WriteString(L" = "); LogString(node->value, writer); writer.WriteString(L" }"); if(parentPriority>priority) { writer.WriteString(L" )"); } } }; void Log(ParsingDefinitionGrammar* grammar, TextWriter& writer) { ParsingDefinitionGrammarLogger::LogInternal(grammar, PRIORITY_NONE, 0, true, writer); } void Log(ParsingDefinitionGrammar* grammar, ParsingDefinitionGrammar* stateNode, bool beforeNode, TextWriter& writer) { ParsingDefinitionGrammarLogger::LogInternal(grammar, PRIORITY_NONE, stateNode, beforeNode, writer); } #undef PRIORITY_NONE #undef PRIORITY_CREATE #undef PRIORITY_SET #undef PRIORITY_ALTERNATIVE #undef PRIORITY_SEQUENCE #undef PRIORITY_USE #undef PRIORITY_ASSIGN /*********************************************************************** FindAppropriateGrammarState ***********************************************************************/ class FindAppropriateGrammarStateVisitor : public Object, public ParsingDefinitionGrammar::IVisitor { public: ParsingDefinitionGrammar* stateNode; bool beforeNode; ParsingDefinitionGrammar* beforeReference; ParsingDefinitionGrammar* afterReference; ParsingDefinitionGrammar* result; FindAppropriateGrammarStateVisitor(ParsingDefinitionGrammar* _stateNode, bool _beforeNode, ParsingDefinitionGrammar* _beforeReference, ParsingDefinitionGrammar* _afterReference) :stateNode(_stateNode) ,beforeNode(_beforeNode) ,beforeReference(_beforeReference) ,afterReference(_afterReference) ,result(0) { } static ParsingDefinitionGrammar* Find(ParsingDefinitionGrammar* grammar, ParsingDefinitionGrammar* stateNode, bool beforeNode, ParsingDefinitionGrammar* beforeReference, ParsingDefinitionGrammar* afterReference) { if(grammar==stateNode) { return beforeNode ?(beforeReference?beforeReference:stateNode) :(afterReference?afterReference:stateNode) ; } else { FindAppropriateGrammarStateVisitor visitor(stateNode, beforeNode, beforeReference, afterReference); grammar->Accept(&visitor); return visitor.result; } } void Visit(ParsingDefinitionPrimitiveGrammar* node)override { } void Visit(ParsingDefinitionTextGrammar* node)override { } void Visit(ParsingDefinitionSequenceGrammar* node)override { result=Find(node->first.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), 0); if(!result) { result=Find(node->second.Obj(), stateNode, beforeNode, 0, (afterReference?afterReference:node)); } } void Visit(ParsingDefinitionAlternativeGrammar* node)override { result=Find(node->first.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); if(!result) { result=Find(node->second.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); } } void Visit(ParsingDefinitionLoopGrammar* node)override { result=Find(node->grammar.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); } void Visit(ParsingDefinitionOptionalGrammar* node)override { result=Find(node->grammar.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); } void Visit(ParsingDefinitionCreateGrammar* node)override { result=Find(node->grammar.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); } void Visit(ParsingDefinitionAssignGrammar* node)override { result=Find(node->grammar.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); } void Visit(ParsingDefinitionUseGrammar* node)override { result=Find(node->grammar.Obj(), stateNode, beforeNode, (beforeReference?beforeReference:node), (afterReference?afterReference:node)); } void Visit(ParsingDefinitionSetterGrammar* node)override { result=Find(node->grammar.Obj(), stateNode, beforeNode, beforeReference, afterReference); } }; /*********************************************************************** Logger (ParsingDefinitionGrammar) ***********************************************************************/ WString TypeToString(ParsingDefinitionType* type) { return GenerateToStream([&](StreamWriter& writer) { Log(type, writer); }, 64); } WString GrammarToString(ParsingDefinitionGrammar* grammar) { return GrammarStateToString(grammar, 0, true); } WString GrammarStateToString(ParsingDefinitionGrammar* grammar, ParsingDefinitionGrammar* stateNode, bool beforeNode) { return GenerateToStream([&](StreamWriter& writer) { Log(grammar, stateNode, beforeNode, writer); }, 64); } ParsingDefinitionGrammar* FindAppropriateGrammarState(ParsingDefinitionGrammar* grammar, ParsingDefinitionGrammar* stateNode, bool beforeNode) { return FindAppropriateGrammarStateVisitor::Find(grammar, stateNode, beforeNode, 0, 0); } void Log(Ptr definition, TextWriter& writer) { for (auto type : definition->types) { Log(type.Obj(), L"", writer); writer.WriteLine(L""); } for (auto token : definition->tokens) { if(token->discard) { writer.WriteString(L"discardtoken "); } else { writer.WriteString(L"token "); } writer.WriteString(token->name); writer.WriteString(L" = "); LogString(token->regex, writer); LogAttributeList(token.Obj(), writer); writer.WriteLine(L";"); } writer.WriteLine(L""); for (auto rule : definition->rules) { writer.WriteString(L"rule "); Log(rule->type.Obj(), writer); writer.WriteString(L" "); writer.WriteString(rule->name); LogAttributeList(rule.Obj(), writer); writer.WriteLine(L""); for (auto grammar : rule->grammars) { writer.WriteString(L" = "); Log(grammar.Obj(), writer); writer.WriteLine(L""); } writer.WriteLine(L" ;"); } } } namespace analyzing { /*********************************************************************** Logger (Automaton) ***********************************************************************/ void LogTransitionSymbol(ParsingSymbol* symbol, stream::TextWriter& writer) { if (symbol->GetType() == ParsingSymbol::TokenDef) { writer.WriteString(L"["); writer.WriteString(symbol->GetName()); U32String regex = wtou32(symbol->GetDescriptorString()); if (regex_internal::IsRegexEscapedLiteralString(regex)) { writer.WriteString(L" "); definitions::LogString(u32tow(regex_internal::UnescapeTextForRegex(regex)), writer); } writer.WriteString(L"]"); } else { writer.WriteString(L"<"); writer.WriteString(symbol->GetName()); writer.WriteString(L">"); } } void Log(Ptr automaton, stream::TextWriter& writer) { for (auto ruleInfo : automaton->ruleInfos) { writer.WriteString(L"Root Rule Start: "); writer.WriteLine(ruleInfo->rootRuleStartState->stateName); writer.WriteString(L"Root Rule End: "); writer.WriteLine(ruleInfo->rootRuleEndState->stateName); writer.WriteString(L"Rule Start: "); writer.WriteLine(ruleInfo->startState->stateName); for (auto endState : ruleInfo->endStates) { writer.WriteString(L"Rule End: "); writer.WriteLine(endState->stateName); } writer.WriteLine(L""); } List states; for (auto ruleInfo : automaton->ruleInfos) { vint currentState=states.Count(); states.Add(ruleInfo->rootRuleStartState); while(currentStatestateExpression); if(state->endState) { writer.WriteString(L"END STATE "); } else { writer.WriteString(L"STATE "); } writer.WriteLine(state->stateName); for (auto transition : state->transitions) { if(!states.Contains(transition->target)) { states.Add(transition->target); } switch(transition->transitionType) { case Transition::Epsilon: writer.WriteString(L" EPSILON"); break; case Transition::TokenBegin: writer.WriteString(L" TOKEN-BEGIN"); break; case Transition::TokenFinish: writer.WriteString(L" TOKEN-FINISH"); break; case Transition::NormalReduce: writer.WriteString(L" NORMAL-REDUCE"); break; case Transition::LeftRecursiveReduce: writer.WriteString(L" LREC-REDUCE"); break; case Transition::Symbol: { writer.WriteString(L" "); if(transition->transitionSymbol) { LogTransitionSymbol(transition->transitionSymbol, writer); } } break; } switch(transition->stackOperationType) { case Transition::None: writer.WriteString(L" => "); break; case Transition::ShiftReduceCompacted: writer.WriteString(L" [SHIFT-REDUCE-COMPACTED] => "); break; case Transition::LeftRecursive: writer.WriteString(L" [LEFT-RECURSIVE] => "); break; } writer.WriteLine(transition->target->stateName); for (auto action : transition->actions) { switch(action->actionType) { case Action::Create: writer.WriteString(L" CREATE "); break; case Action::Assign: writer.WriteString(L" ASSIGN "); break; case Action::Using: writer.WriteString(L" USING "); break; case Action::Setter: writer.WriteString(L" SET "); break; case Action::Shift: writer.WriteString(L" SHIFT "); break; case Action::Reduce: writer.WriteString(L" REDUCE "); break; case Action::LeftRecursiveReduce: writer.WriteString(L" LR-REDUCE "); break; } if(action->shiftReduceSource && action->shiftReduceTarget) { writer.WriteString(L"["); writer.WriteString(action->shiftReduceSource->stateName); writer.WriteString(L" => "); writer.WriteString(action->shiftReduceTarget->stateName); writer.WriteString(L"] "); } if(action->actionSource) { writer.WriteString(action->actionSource->GetName()); } if(action->actionTarget) { writer.WriteString(L" => "); writer.WriteString(action->actionTarget->GetName()); } writer.WriteLine(L""); } } writer.WriteLine(L""); } writer.WriteLine(L"--------------------------------"); } } } namespace tabling { /*********************************************************************** Logger (ParsingTable) ***********************************************************************/ void LogAttributeList(Ptr table, vint attributeIndex, const WString& prefix, stream::TextWriter& writer) { if(attributeIndex!=-1) { Ptr atts=table->GetAttributeInfo(attributeIndex); for (auto att : atts->attributes) { writer.WriteString(prefix); writer.WriteString(L"@"); writer.WriteString(att->name); writer.WriteString(L"("); for(vint i=0;iarguments.Count();i++) { if(i>0) writer.WriteString(L", "); definitions::LogString(att->arguments[i], writer); } writer.WriteLine(L")"); } } } void Log(Ptr table, stream::TextWriter& writer) { vint rows=table->GetStateCount()+1; vint columns=table->GetTokenCount()+1; Array stringTable(rows*columns); stringTable[0]=L""; for(vint row=0; rowGetStateCount();row++) { stringTable[(row+1)*columns]=itow(row)+L": "+table->GetStateInfo(row).stateName; } for(vint column=0;columnGetTokenCount();column++) { WString content= column==ParsingTable::TokenBegin?L"0: $TokenBegin": column==ParsingTable::TokenFinish?L"1: $TokenFinish": column==ParsingTable::NormalReduce?L"2: $NormalReduce": column==ParsingTable::LeftRecursiveReduce?L"3: $LeftRecursiveReduce": itow(column)+L": "+table->GetTokenInfo(column).name+L"\r\n "+table->GetTokenInfo(column).regex; stringTable[column+1]=content; } for(vint row=0; rowGetStateCount();row++) { for(vint column=0;columnGetTokenCount();column++) { Ptr bag=table->GetTransitionBag(row, column); if(bag) { WString content; for (auto item : bag->transitionItems) { if(content!=L"") content+=L"\r\n"; content+=itow(item->targetState); for (auto [state, index] : indexed(item->stackPattern)) { content+=(index==0?L" : ":L", "); content+=itow(state); } content+=L"\r\n"; for (auto lookAhead : item->lookAheads) { content+=L" "; for (auto [token, index] : indexed(lookAhead->tokens)) { content+=(index==0?L"> ":L", "); content+=itow(token); } content+=L"\r\n"; } content+=L" "; for (auto ins : item->instructions) { switch(ins.instructionType) { case ParsingTable::Instruction::Create: content+=L"C"; break; case ParsingTable::Instruction::Using: content+=L"U"; break; case ParsingTable::Instruction::Assign: content+=L"A"; break; case ParsingTable::Instruction::Item: content+=L"I"; break; case ParsingTable::Instruction::Setter: content+=L"S"; break; case ParsingTable::Instruction::Shift: content+=L"[+"+itow(ins.stateParameter)+L"]"; break; case ParsingTable::Instruction::Reduce: content+=L"[-"+itow(ins.stateParameter)+L"]"; break; case ParsingTable::Instruction::LeftRecursiveReduce: content+=L"[!"+itow(ins.stateParameter)+L"]"; break; } } } stringTable[(row+1)*columns+(column+1)]=content; } } } writer.WriteLine(L"Target-State : Stack-Pattern ..."); writer.WriteLine(L"> Look-Ahead ..."); writer.WriteLine(L"C: Create"); writer.WriteLine(L"U: Using"); writer.WriteLine(L"A: Assign"); writer.WriteLine(L"I: Item"); writer.WriteLine(L"S: Setter"); writer.WriteLine(L"[+s]: Shift[push s]"); writer.WriteLine(L"[-s]: Reduce[pop s]"); writer.WriteLine(L"[!s]: Left-Recursive-Reduce[fake s]"); writer.WriteLine(L""); for(vint i=0;iGetRuleCount();i++) { const ParsingTable::RuleInfo& ruleInfo=table->GetRuleInfo(i); writer.WriteString(ruleInfo.name); writer.WriteChar(L'<'); writer.WriteString(ruleInfo.type); if(ruleInfo.ambiguousType!=L"") { writer.WriteString(L", "); writer.WriteString(ruleInfo.ambiguousType); } writer.WriteString(L">: "); writer.WriteString(itow(ruleInfo.rootStartState)); writer.WriteChar(L'['); writer.WriteString(table->GetStateInfo(ruleInfo.rootStartState).stateName); writer.WriteChar(L']'); writer.WriteLine(L""); } WriteMonospacedEnglishTable(writer, stringTable, rows, columns); writer.WriteLine(L""); writer.WriteLine(L"Metadata(Tokens):"); for(vint i=0;iGetTokenCount();i++) { const ParsingTable::TokenInfo& info=table->GetTokenInfo(i); writer.WriteString(L" "); writer.WriteString(info.name); writer.WriteString(L"="); writer.WriteLine(info.regex); LogAttributeList(table, info.attributeIndex, L" ", writer); } writer.WriteLine(L""); writer.WriteLine(L"Metadata(Rules):"); for(vint i=0;iGetRuleCount();i++) { const ParsingTable::RuleInfo& info=table->GetRuleInfo(i); writer.WriteString(L" "); writer.WriteLine(info.name); LogAttributeList(table, info.attributeIndex, L" ", writer); } writer.WriteLine(L""); writer.WriteLine(L"Metadata(Classes):"); for(vint i=0;iGetTreeTypeInfoCount();i++) { const ParsingTable::TreeTypeInfo& info=table->GetTreeTypeInfo(i); writer.WriteString(L" "); writer.WriteLine(info.type); LogAttributeList(table, info.attributeIndex, L" ", writer); } writer.WriteLine(L""); writer.WriteLine(L"Metadata(Class Members):"); for(vint i=0;iGetTreeFieldInfoCount();i++) { const ParsingTable::TreeFieldInfo& info=table->GetTreeFieldInfo(i); writer.WriteString(L" "); writer.WriteString(info.type); writer.WriteString(L"."); writer.WriteLine(info.field); LogAttributeList(table, info.attributeIndex, L" ", writer); } writer.WriteLine(L""); } } /*********************************************************************** Logger (ParsingTreeNode) ***********************************************************************/ class LogParsingTreeNodeVisitor : public Object, public ParsingTreeNode::IVisitor { protected: TextWriter& writer; WString prefix; WString originalInput; public: LogParsingTreeNodeVisitor(TextWriter& _writer, const WString& _originalInput, const WString& _prefix) :writer(_writer) ,prefix(_prefix) ,originalInput(_originalInput) { } void Write(ParsingTreeNode* node) { if(node) { node->Accept(this); } else { writer.WriteString(L"null"); } } void WriteInput(ParsingTreeNode* node) { if(originalInput!=L"") { ParsingTextRange range=node->GetCodeRange(); if(range.start.index!=ParsingTextPos::UnknownValue && range.end.index!=ParsingTextPos::UnknownValue) { vint start=range.start.index; vint length=range.end.index-start+1; if(length>0) { writer.WriteString(L" // ["); writer.WriteString(originalInput.Sub(start, length)); writer.WriteString(L"]"); } } } } void Visit(ParsingTreeToken* node) { writer.WriteChar(L'['); writer.WriteString(node->GetValue()); writer.WriteChar(L']'); WriteInput(node); } void Visit(ParsingTreeObject* node) { WString oldPrefix=prefix; writer.WriteString(node->GetType()); writer.WriteString(L" <"); for(vint i=0;iGetCreatorRules().Count();i++) { if(i!=0) writer.WriteString(L", "); writer.WriteString(node->GetCreatorRules()[i]); } writer.WriteString(L"> {"); WriteInput(node); writer.WriteLine(L""); prefix+=L" "; for(vint i=0;iGetMembers().Count();i++) { writer.WriteString(prefix); writer.WriteString(node->GetMembers().Keys().Get(i)); writer.WriteString(L" = "); Write(node->GetMembers().Values().Get(i).Obj()); writer.WriteLine(L""); } prefix=oldPrefix; writer.WriteString(prefix); writer.WriteString(L"}"); } void Visit(ParsingTreeArray* node) { WString oldPrefix=prefix; writer.WriteString(node->GetElementType()); writer.WriteString(L"[] {"); WriteInput(node); writer.WriteLine(L""); prefix+=L" "; for(vint i=0;iCount();i++) { writer.WriteString(prefix); Write(node->GetItem(i).Obj()); writer.WriteLine(L","); } prefix=oldPrefix; writer.WriteString(prefix); writer.WriteString(L"}"); } }; void Log(ParsingTreeNode* node, const WString& originalInput, stream::TextWriter& writer, const WString& prefix) { writer.WriteString(prefix); LogParsingTreeNodeVisitor visitor(writer, originalInput, prefix); node->Accept(&visitor); } } } /*********************************************************************** .\PARSINGSTATE.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ #if defined(VCZH_GCC) && defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wnull-dereference" #endif namespace vl { namespace parsing { namespace tabling { using namespace collections; using namespace regex; /*********************************************************************** ParsingTokenWalker::LookAheadEnumerator ***********************************************************************/ ParsingTokenWalker::LookAheadEnumerator::LookAheadEnumerator(const ParsingTokenWalker* _walker, vint _currentToken) :walker(_walker) ,firstToken(_currentToken) ,currentToken(_currentToken) ,currentValue(-1) ,index(-1) { } ParsingTokenWalker::LookAheadEnumerator::LookAheadEnumerator(const LookAheadEnumerator& _enumerator) :walker(_enumerator.walker) ,firstToken(_enumerator.firstToken) ,currentToken(_enumerator.currentToken) ,currentValue(_enumerator.currentValue) ,index(_enumerator.index) { } IEnumerator* ParsingTokenWalker::LookAheadEnumerator::Clone()const { return new LookAheadEnumerator(*this); } const vint& ParsingTokenWalker::LookAheadEnumerator::Current()const { return currentValue; } vint ParsingTokenWalker::LookAheadEnumerator::Index()const { return index; } bool ParsingTokenWalker::LookAheadEnumerator::Next() { vint newToken=walker->GetNextIndex(currentToken); if(newToken==-3) return false; currentToken=newToken; index++; currentValue=walker->GetTableTokenIndex(currentToken); return true; } void ParsingTokenWalker::LookAheadEnumerator::Reset() { currentToken=firstToken; currentValue=-1; index=-1; } /*********************************************************************** ParsingTokenWalker::TokenLookAhead ***********************************************************************/ ParsingTokenWalker::TokenLookAhead::TokenLookAhead(const ParsingTokenWalker* _walker) :walker(_walker) { } collections::IEnumerator* ParsingTokenWalker::TokenLookAhead::CreateEnumerator()const { return new LookAheadEnumerator(walker, walker->currentToken); } /*********************************************************************** ParsingTokenWalker::ReduceLookAhead ***********************************************************************/ ParsingTokenWalker::ReduceLookAhead::ReduceLookAhead(const ParsingTokenWalker* _walker) :walker(_walker) { } collections::IEnumerator* ParsingTokenWalker::ReduceLookAhead::CreateEnumerator()const { return new LookAheadEnumerator(walker, walker->currentToken - 1); } /*********************************************************************** ParsingTokenWalker ***********************************************************************/ vint ParsingTokenWalker::GetNextIndex(vint index)const { if(index==-2) { return -1; } else if(-1<=index && indexIsInputToken(tokens[index].token)) { break; } else { index++; } } return index; } else { return -3; } } vint ParsingTokenWalker::GetTableTokenIndex(vint index)const { if(index==-1) { return ParsingTable::TokenBegin; } else if(index==tokens.Count()) { return ParsingTable::TokenFinish; } else if(0<=index && indexGetTableTokenIndex(tokens[index].token); } else { return -1; } } ParsingTokenWalker::ParsingTokenWalker(collections::List& _tokens, Ptr _table) :tokens(_tokens) ,table(_table) ,currentToken(-2) , tokenLookAhead(this) , reduceLookAhead(this) { } ParsingTokenWalker::~ParsingTokenWalker() { } const collections::IEnumerable& ParsingTokenWalker::GetTokenLookahead()const { return tokenLookAhead; } const collections::IEnumerable& ParsingTokenWalker::GetReduceLookahead()const { return reduceLookAhead; } void ParsingTokenWalker::Reset() { currentToken=-2; } bool ParsingTokenWalker::Move() { currentToken=GetNextIndex(currentToken); return currentToken!=-3; } vint ParsingTokenWalker::GetTableTokenIndex()const { return GetTableTokenIndex(currentToken); } regex::RegexToken* ParsingTokenWalker::GetRegexToken()const { vint index=GetTokenIndexInStream(); return index==-1?0:&tokens[index]; } vint ParsingTokenWalker::GetTokenIndexInStream()const { if(0<=currentToken && currentToken _table, vint codeIndex) :input(_input.Buffer()) ,table(_table) ,parsingRuleStartState(-1) { CopyFrom(tokens, table->GetLexer().Parse(input, {}, codeIndex)); walker = Ptr(new ParsingTokenWalker(tokens, table)); } ParsingState::~ParsingState() { } const WString& ParsingState::GetInput() { return input; } Ptr ParsingState::GetTable() { return table; } const collections::List& ParsingState::GetTokens() { return tokens; } regex::RegexToken* ParsingState::GetToken(vint index) { if(index<=0) { index=0; } else if(index>tokens.Count()) { index=tokens.Count(); } return index==tokens.Count()?0:&tokens[index]; } vint ParsingState::Reset(const WString& rule) { const ParsingTable::RuleInfo& info=table->GetRuleInfo(rule); auto infoExists = &info; if(infoExists) { walker->Reset(); walker->Move(); stateGroup = Ptr(new StateGroup(info)); parsingRule=rule; parsingRuleStartState=info.rootStartState; return stateGroup->currentState; } return -1; } WString ParsingState::GetParsingRule() { return parsingRule; } vint ParsingState::GetParsingRuleStartState() { return parsingRuleStartState; } vint ParsingState::GetCurrentToken() { return walker->GetTableTokenIndex()==ParsingTable::TokenFinish ?tokens.Count() :walker->GetTokenIndexInStream(); } vint ParsingState::GetCurrentTableTokenIndex() { return walker->GetTableTokenIndex(); } const collections::List& ParsingState::GetStateStack() { return stateGroup->stateStack; } vint ParsingState::GetCurrentState() { return stateGroup->currentState; } void ParsingState::SkipCurrentToken() { walker->Move(); } bool ParsingState::TestTransitionItemInFuture(vint tableTokenIndex, Future* future, ParsingTable::TransitionItem* item, const collections::IEnumerable* lookAheadTokens) { bool passLookAheadTest=true; if(item->lookAheads.Count()>0 && lookAheadTokens) { passLookAheadTest=false; for (auto info : item->lookAheads) { vint index=0; for (auto token : *lookAheadTokens) { if(info->tokens[index]!=token) { break; } index++; if(index>=info->tokens.Count()) { break; } } if(index==info->tokens.Count()) { passLookAheadTest=true; break; } } } if(!passLookAheadTest) { return false; } vint availableStackDepth=stateGroup->stateStack.Count()-future->reduceStateCount; vint totalStackDepth=stateGroup->stateStack.Count()-future->reduceStateCount+future->shiftStates.Count(); if(item->stackPattern.Count()<=totalStackDepth) { if(tableTokenIndex!=ParsingTable::TokenFinish || item->stackPattern.Count()==totalStackDepth) { bool match=true; for(vint j=0;jstackPattern.Count();j++) { vint state= jshiftStates.Count() ?future->shiftStates[future->shiftStates.Count()-1-j] :stateGroup->stateStack[availableStackDepth-1-(j-future->shiftStates.Count())] ; if(item->stackPattern[j]!=state) { match=false; } } if(match) { return true; } } } return false; } ParsingTable::TransitionItem* ParsingState::MatchTokenInFuture(vint tableTokenIndex, Future* future, const collections::IEnumerable* lookAheadTokens) { ParsingTable::TransitionBag* bag=table->GetTransitionBag(future->currentState, tableTokenIndex).Obj(); if(bag) { for(vint i=0;itransitionItems.Count();i++) { ParsingTable::TransitionItem* item=bag->transitionItems[i].Obj(); if(TestTransitionItemInFuture(tableTokenIndex, future, item, lookAheadTokens)) { return item; } } } return 0; } ParsingTable::TransitionItem* ParsingState::MatchToken(vint tableTokenIndex, const collections::IEnumerable* lookAheadTokens) { Future future; future.currentState=stateGroup->currentState; return MatchTokenInFuture(tableTokenIndex, &future, lookAheadTokens); } void ParsingState::RunTransitionInFuture(ParsingTable::TransitionItem* transition, Future* previous, Future* now) { if(previous) { now->reduceStateCount=previous->reduceStateCount; CopyFrom(now->shiftStates, previous->shiftStates); } else { now->reduceStateCount=0; now->shiftStates.Clear(); } now->currentState=transition->targetState; now->selectedToken=transition->token; now->selectedItem=transition; now->previous=previous; now->next=0; for(vint j=0;jinstructions.Count();j++) { ParsingTable::Instruction& ins=transition->instructions[j]; switch(ins.instructionType) { case ParsingTable::Instruction::Shift: { now->shiftStates.Add(ins.stateParameter); } break; case ParsingTable::Instruction::Reduce: { if(now->shiftStates.Count()==0) { now->reduceStateCount++; } else { now->shiftStates.RemoveAt(now->shiftStates.Count()-1); } } break; default:; } } } ParsingState::TransitionResult ParsingState::RunTransition(ParsingTable::TransitionItem* transition, regex::RegexToken* regexToken, vint instructionBegin, vint instructionCount, bool lastPart) { if(regexToken) { if(!stateGroup->shiftToken) { stateGroup->shiftToken=regexToken; stateGroup->reduceToken=regexToken; } } if(transition->token>=ParsingTable::UserTokenStart) { if(stateGroup->tokenSequenceIndex==0) { stateGroup->shiftTokenStack.Add(stateGroup->shiftToken); } stateGroup->tokenSequenceIndex++; } TransitionResult result; result.tableTokenIndex=transition->token; result.token=regexToken; result.tokenIndexInStream=regexToken?walker->GetTokenIndexInStream():-1; result.tableStateSource=stateGroup->currentState; result.tableStateTarget=transition->targetState; result.transition=transition; result.instructionBegin=instructionBegin; result.instructionCount=instructionCount; for(vint j=instructionBegin;jinstructions[j]; switch(ins.instructionType) { case ParsingTable::Instruction::Shift: { stateGroup->stateStack.Add(ins.stateParameter); stateGroup->shiftTokenStack.Add(stateGroup->shiftToken); stateGroup->shiftToken=regexToken; stateGroup->reduceToken=regexToken; } break; case ParsingTable::Instruction::Reduce: { stateGroup->stateStack.RemoveAt(stateGroup->stateStack.Count()-1); result.AddShiftReduceRange(stateGroup->shiftToken, stateGroup->reduceToken); stateGroup->shiftToken=stateGroup->shiftTokenStack[stateGroup->shiftTokenStack.Count()-1]; stateGroup->shiftTokenStack.RemoveAt(stateGroup->shiftTokenStack.Count()-1); } break; case ParsingTable::Instruction::LeftRecursiveReduce: { result.AddShiftReduceRange(stateGroup->shiftToken, stateGroup->reduceToken); if(regexToken) { stateGroup->reduceToken=regexToken; } } break; default:; } } if(regexToken) { stateGroup->reduceToken=regexToken; } if(transition->token==ParsingTable::TokenFinish && lastPart) { stateGroup->shiftToken=stateGroup->shiftTokenStack[stateGroup->shiftTokenStack.Count()-1]; stateGroup->shiftTokenStack.RemoveAt(stateGroup->shiftTokenStack.Count()-1); result.AddShiftReduceRange(stateGroup->shiftToken, stateGroup->reduceToken); } stateGroup->currentState=transition->targetState; return result; } ParsingState::TransitionResult ParsingState::RunTransition(ParsingTable::TransitionItem* transition, regex::RegexToken* regexToken) { return RunTransition(transition, regexToken, 0, transition->instructions.Count(), true); } bool ParsingState::ReadTokenInFuture(vint tableTokenIndex, Future* previous, Future* now, const collections::IEnumerable* lookAheadTokens) { ParsingTable::TransitionItem* selectedItem=0; if(previous) { selectedItem=MatchTokenInFuture(tableTokenIndex, previous, lookAheadTokens); } else { selectedItem=MatchToken(tableTokenIndex, lookAheadTokens); } if(!selectedItem) { return false; } RunTransitionInFuture(selectedItem, previous, now); return true; } ParsingState::TransitionResult ParsingState::ReadToken(vint tableTokenIndex, regex::RegexToken* regexToken, const collections::IEnumerable* lookAheadTokens) { ParsingTable::TransitionItem* item=MatchToken(tableTokenIndex, lookAheadTokens); if(item) { return RunTransition(item, regexToken); } return TransitionResult(); } ParsingState::TransitionResult ParsingState::ReadToken() { if(walker->GetTableTokenIndex()==-1) { return TransitionResult(); } vint token=walker->GetTableTokenIndex(); RegexToken* regexToken=walker->GetRegexToken(); bool tryReduce=false; TransitionResult result=ReadToken(token, regexToken, &walker->GetTokenLookahead()); if(!result) { result=ReadToken(ParsingTable::LeftRecursiveReduce, 0, &walker->GetReduceLookahead()); tryReduce=true; } if(!result) { result=ReadToken(ParsingTable::NormalReduce, 0, &walker->GetReduceLookahead()); tryReduce=true; } if(result && !tryReduce) { walker->Move(); } return result; } bool ParsingState::TestExplore(vint tableTokenIndex, Future* previous) { Future fakePrevious; fakePrevious.currentState=stateGroup->currentState; Future* realPrevious=previous?previous:&fakePrevious; ParsingTable::TransitionBag* bag=table->GetTransitionBag(realPrevious->currentState, tableTokenIndex).Obj(); if(bag) { for(vint i=0;itransitionItems.Count();i++) { ParsingTable::TransitionItem* item=bag->transitionItems[i].Obj(); if(TestTransitionItemInFuture(tableTokenIndex, realPrevious, item, 0)) { return true; } } } return false; } bool ParsingState::Explore(vint tableTokenIndex, Future* previous, collections::List& possibilities) { Future fakePrevious; fakePrevious.currentState=stateGroup->currentState; Future* realPrevious=previous?previous:&fakePrevious; ParsingTable::TransitionBag* bag=table->GetTransitionBag(realPrevious->currentState, tableTokenIndex).Obj(); bool successful = false; if(bag) { for(vint i=0;itransitionItems.Count();i++) { ParsingTable::TransitionItem* item=bag->transitionItems[i].Obj(); if(TestTransitionItemInFuture(tableTokenIndex, realPrevious, item, 0)) { Future* now=new Future; RunTransitionInFuture(item, previous, now); possibilities.Add(now); successful = true; } } } return successful; } bool ParsingState::ExploreStep(collections::List& previousFutures, vint start, vint count, collections::List& possibilities) { if(walker->GetTableTokenIndex()==-1) { return false; } vint token = walker->GetTableTokenIndex(); RegexToken* regexToken = walker->GetRegexToken(); vint oldPossibilitiesCount = possibilities.Count(); for (vint i = 0; iselectedRegexToken = regexToken; } return true; } bool ParsingState::ExploreNormalReduce(collections::List& previousFutures, vint start, vint count, collections::List& possibilities) { if(walker->GetTableTokenIndex()==-1) { return false; } vint oldPossibilitiesCount = possibilities.Count(); for(vint i=0;i oldPossibilitiesCount; } bool ParsingState::ExploreLeftRecursiveReduce(collections::List& previousFutures, vint start, vint count, collections::List& possibilities) { if(walker->GetTableTokenIndex()==-1) { return false; } vint oldPossibilitiesCount = possibilities.Count(); for(vint i=0;i oldPossibilitiesCount; } ParsingState::Future* ParsingState::ExploreCreateRootFuture() { Future* future=new Future; future->currentState=stateGroup->currentState; return future; } Ptr ParsingState::TakeSnapshot() { return Ptr(new StateGroup(*stateGroup.Obj())); } void ParsingState::RestoreSnapshot(Ptr group) { stateGroup = Ptr(new StateGroup(*group.Obj())); } /*********************************************************************** ParsingTreeBuilder ***********************************************************************/ ParsingTreeBuilder::ParsingTreeBuilder() :processingAmbiguityBranch(false) ,ambiguityBranchSharedNodeCount(0) { } ParsingTreeBuilder::~ParsingTreeBuilder() { } void ParsingTreeBuilder::Reset() { createdObject=0; operationTarget = Ptr(new ParsingTreeObject()); nodeStack.Clear(); processingAmbiguityBranch=false; ambiguityBranchCreatedObject=0; ambiguityBranchOperationTarget=0; ambiguityBranchSharedNodeCount=0; ambiguityBranchNodeStack.Clear(); ambiguityNodes.Clear(); } bool ParsingTreeBuilder::Run(const ParsingState::TransitionResult& result) { if(!operationTarget) { return false; } switch(result.transitionType) { case ParsingState::TransitionResult::AmbiguityBegin: { if(processingAmbiguityBranch) return false; processingAmbiguityBranch=true; if(createdObject) { ambiguityBranchCreatedObject=createdObject->Clone(); } else { ambiguityBranchCreatedObject=0; } ambiguityBranchOperationTarget=operationTarget->Clone().Cast(); ambiguityBranchNodeStack.Clear(); ambiguityBranchSharedNodeCount=nodeStack.Count()-result.ambiguityAffectedStackNodeCount+1; for(vint i=ambiguityBranchSharedNodeCount;iClone().Cast()); } ambiguityNodes.Clear(); } break; case ParsingState::TransitionResult::AmbiguityBranch: { if(!processingAmbiguityBranch) return false; if(nodeStack.Count()!=ambiguityBranchSharedNodeCount) return false; ambiguityNodes.Add(operationTarget); if(ambiguityBranchCreatedObject) { createdObject=ambiguityBranchCreatedObject->Clone(); } else { createdObject=0; } operationTarget=ambiguityBranchOperationTarget->Clone().Cast(); for(vint i=0;iClone().Cast()); } } break; case ParsingState::TransitionResult::AmbiguityEnd: { if(!processingAmbiguityBranch) return false; if(nodeStack.Count()!=ambiguityBranchSharedNodeCount) return false; ambiguityNodes.Add(operationTarget); processingAmbiguityBranch=false; createdObject=0; ambiguityBranchCreatedObject=0; ambiguityBranchOperationTarget=0; ambiguityBranchSharedNodeCount=0; ambiguityBranchNodeStack.Clear(); { auto ambiguousNode = Ptr(new ParsingTreeObject(result.ambiguityNodeType, operationTarget->GetCodeRange())); auto items = Ptr(new ParsingTreeArray(L"", operationTarget->GetCodeRange())); for (auto node : ambiguityNodes) { items->AddItem(node); } ambiguousNode->SetMember(L"items", items); operationTarget=ambiguousNode; } ambiguityNodes.Clear(); } break; case ParsingState::TransitionResult::ExecuteInstructions: { vint shiftReduceRangeIndex=0; for(vint j=result.instructionBegin;jinstructions[j]; switch(ins.instructionType) { case ParsingTable::Instruction::Create: { if(operationTarget->GetType()!=L"") { return false; } operationTarget->SetType(ins.nameParameter); operationTarget->GetCreatorRules().Add(ins.creatorRule); } break; case ParsingTable::Instruction::Using: { if(operationTarget->GetType()!=L"" || !createdObject) { return false; } Ptr obj=createdObject.Cast(); if(!obj) { return false; } for(vint i=0;iGetMembers().Count();i++) { WString name=operationTarget->GetMembers().Keys().Get(i); Ptr value=operationTarget->GetMembers().Values().Get(i); obj->SetMember(name, value); } operationTarget=obj; operationTarget->GetCreatorRules().Add(ins.creatorRule); createdObject=0; } break; case ParsingTable::Instruction::Assign: { if(!createdObject) { Ptr value; if(result.token==0) { value = Ptr(new ParsingTreeToken(L"", result.tokenIndexInStream)); } else { value = Ptr(new ParsingTreeToken(WString::CopyFrom(result.token->reading, result.token->length), result.tokenIndexInStream)); value->SetCodeRange(ParsingTextRange(result.token, result.token)); } operationTarget->SetMember(ins.nameParameter, value); } else { operationTarget->SetMember(ins.nameParameter, createdObject); createdObject=0; } } break; case ParsingTable::Instruction::Item: { auto arr=operationTarget->GetMember(ins.nameParameter).Cast(); if(!arr) { arr = Ptr(new ParsingTreeArray()); operationTarget->SetMember(ins.nameParameter, arr); } ParsingTextRange arrRange=arr->GetCodeRange(); ParsingTextRange itemRange; if(!createdObject) { Ptr value; if(result.token==0) { value = Ptr(new ParsingTreeToken(L"", result.tokenIndexInStream)); } else { value = Ptr(new ParsingTreeToken(WString::CopyFrom(result.token->reading, result.token->length), result.tokenIndexInStream)); value->SetCodeRange(ParsingTextRange(result.token, result.token)); itemRange=value->GetCodeRange(); } arr->AddItem(value); } else { arr->AddItem(createdObject); itemRange=createdObject->GetCodeRange(); createdObject=0; } if(arrRange.start.index==ParsingTextPos::UnknownValue || itemRange.startarrRange.end) { arrRange.end=itemRange.end; } arr->SetCodeRange(arrRange); } break; case ParsingTable::Instruction::Setter: { auto value = Ptr(new ParsingTreeToken(ins.value, -1)); operationTarget->SetMember(ins.nameParameter, value); } break; case ParsingTable::Instruction::Shift: { nodeStack.Add(operationTarget); operationTarget = Ptr(new ParsingTreeObject()); createdObject=0; } break; case ParsingTable::Instruction::Reduce: { if(nodeStack.Count()==0) { return false; } createdObject=operationTarget; operationTarget=nodeStack[nodeStack.Count()-1]; nodeStack.RemoveAt(nodeStack.Count()-1); if(result.shiftReduceRanges) { ParsingState::ShiftReduceRange tokenRange=result.shiftReduceRanges->Get(shiftReduceRangeIndex++); if(tokenRange.shiftToken && tokenRange.reduceToken) { ParsingTextRange codeRange(tokenRange.shiftToken, tokenRange.reduceToken); createdObject->SetCodeRange(codeRange); } } } break; case ParsingTable::Instruction::LeftRecursiveReduce: { createdObject=operationTarget; operationTarget = Ptr(new ParsingTreeObject()); if(result.shiftReduceRanges) { ParsingState::ShiftReduceRange tokenRange=result.shiftReduceRanges->Get(shiftReduceRangeIndex++); if(tokenRange.shiftToken && tokenRange.reduceToken) { ParsingTextRange codeRange(tokenRange.shiftToken, tokenRange.reduceToken); createdObject->SetCodeRange(codeRange); } } } break; default: return false; } } if(result.tableTokenIndex==ParsingTable::TokenFinish && !processingAmbiguityBranch) { if(result.shiftReduceRanges) { ParsingState::ShiftReduceRange tokenRange=result.shiftReduceRanges->Get(shiftReduceRangeIndex++); if(tokenRange.shiftToken && tokenRange.reduceToken) { ParsingTextRange codeRange(tokenRange.shiftToken, tokenRange.reduceToken); operationTarget->SetCodeRange(codeRange); } } } } break; default: return false; } return true; } bool ParsingTreeBuilder::GetProcessingAmbiguityBranch() { return processingAmbiguityBranch; } Ptr ParsingTreeBuilder::GetNode()const { if(nodeStack.Count()==0) { return operationTarget; } else { return 0; } } /*********************************************************************** ParsingTransitionCollector ***********************************************************************/ ParsingTransitionCollector::ParsingTransitionCollector() :ambiguityBegin(-1) { } ParsingTransitionCollector::~ParsingTransitionCollector() { } void ParsingTransitionCollector::Reset() { ambiguityBegin=-1; transitions.Clear(); ambiguityBeginToEnds.Clear(); ambiguityBeginToBranches.Clear(); ambiguityBranchToBegins.Clear(); } bool ParsingTransitionCollector::Run(const ParsingState::TransitionResult& result) { vint index=transitions.Count(); switch(result.transitionType) { case ParsingState::TransitionResult::AmbiguityBegin: if(ambiguityBegin!=-1) return false; ambiguityBegin=index; break; case ParsingState::TransitionResult::AmbiguityBranch: { if(ambiguityBegin==-1) return false; ambiguityBeginToBranches.Add(ambiguityBegin, index); ambiguityBranchToBegins.Add(index, ambiguityBegin); } break; case ParsingState::TransitionResult::AmbiguityEnd: { if(ambiguityBegin==-1) return false; ambiguityBeginToEnds.Add(ambiguityBegin, index); ambiguityBegin=-1; } break; case ParsingState::TransitionResult::ExecuteInstructions: break; default: return false; } transitions.Add(result); return true; } bool ParsingTransitionCollector::GetProcessingAmbiguityBranch() { return ambiguityBegin!=-1; } const ParsingTransitionCollector::TransitionResultList& ParsingTransitionCollector::GetTransitions()const { return transitions; } vint ParsingTransitionCollector::GetAmbiguityEndFromBegin(vint transitionIndex)const { vint index=ambiguityBeginToEnds.Keys().IndexOf(transitionIndex); return index==-1?-1:ambiguityBeginToEnds.Values()[index]; } const collections::List& ParsingTransitionCollector::GetAmbiguityBranchesFromBegin(vint transitionIndex)const { vint index=ambiguityBeginToBranches.Keys().IndexOf(transitionIndex); return index==-1?*(collections::List*)0:ambiguityBeginToBranches.GetByIndex(index); } vint ParsingTransitionCollector::GetAmbiguityBeginFromBranch(vint transitionIndex)const { vint index=ambiguityBranchToBegins.Keys().IndexOf(transitionIndex); return index==-1?-1:ambiguityBranchToBegins.Values()[index]; } } } } #if defined(VCZH_GCC) && defined(__clang__) #pragma clang diagnostic pop #endif /*********************************************************************** .\PARSINGTABLE.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace stream { namespace internal { using namespace vl::parsing::tabling; /*********************************************************************** ParsingTable (Serialization) ***********************************************************************/ BEGIN_SERIALIZATION(ParsingTable::AttributeInfo) SERIALIZE(name) SERIALIZE(arguments) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::AttributeInfoList) SERIALIZE(attributes) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::TreeTypeInfo) SERIALIZE(type) SERIALIZE(attributeIndex) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::TreeFieldInfo) SERIALIZE(type) SERIALIZE(field) SERIALIZE(attributeIndex) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::TokenInfo) SERIALIZE(name) SERIALIZE(regex) SERIALIZE(regexTokenIndex) SERIALIZE(attributeIndex) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::StateInfo) SERIALIZE(ruleName) SERIALIZE(stateName) SERIALIZE(stateExpression) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::RuleInfo) SERIALIZE(name) SERIALIZE(type) SERIALIZE(ambiguousType) SERIALIZE(rootStartState) SERIALIZE(attributeIndex) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::Instruction) SERIALIZE(instructionType) SERIALIZE(stateParameter) SERIALIZE(nameParameter) SERIALIZE(value) SERIALIZE(creatorRule) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::LookAheadInfo) SERIALIZE(tokens) SERIALIZE(state) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::TransitionItem) SERIALIZE(token) SERIALIZE(targetState) SERIALIZE(lookAheads) SERIALIZE(stackPattern) SERIALIZE(instructions) END_SERIALIZATION BEGIN_SERIALIZATION(ParsingTable::TransitionBag) SERIALIZE(transitionItems) END_SERIALIZATION } } namespace parsing { namespace tabling { using namespace collections; using namespace regex; #ifdef VCZH_GCC const vint ParsingTable::TokenBegin; const vint ParsingTable::TokenFinish; const vint ParsingTable::NormalReduce; const vint ParsingTable::LeftRecursiveReduce; const vint ParsingTable::UserTokenStart; #endif /*********************************************************************** ParsingTable::AttributeInfoList ***********************************************************************/ Ptr ParsingTable::AttributeInfoList::FindFirst(const WString& name) { for(vint i=0;iname==name) { return attributes[i]; } } return 0; } /*********************************************************************** ParsingTable::LookAheadInfo ***********************************************************************/ ParsingTable::LookAheadInfo::PrefixResult ParsingTable::LookAheadInfo::TestPrefix(Ptr a, Ptr b) { if(a->tokens.Count()>b->tokens.Count()) { return ParsingTable::LookAheadInfo::NotPrefix; } for(vint i=0;itokens.Count();i++) { if(a->tokens[i]!=b->tokens[i]) { return ParsingTable::LookAheadInfo::NotPrefix; } } return a->tokens.Count()tokens.Count()?ParsingTable::LookAheadInfo::Prefix:ParsingTable::LookAheadInfo::Equal; } void ParsingTable::LookAheadInfo::WalkInternal(Ptr table, Ptr previous, vint state, collections::SortedList& walkedStates, collections::List>& newInfos) { if (walkedStates.Contains(state)) return; walkedStates.Add(state); for (vint i = 0; i < table->GetTokenCount(); i++) { if(Ptr bag=table->GetTransitionBag(state, i)) { for (auto item : bag->transitionItems) { if (i == ParsingTable::NormalReduce || i == ParsingTable::LeftRecursiveReduce) { WalkInternal(table, previous, item->targetState, walkedStates, newInfos); } else { auto info = Ptr(new LookAheadInfo); info->state=item->targetState; if(previous) { CopyFrom(info->tokens, previous->tokens); } info->tokens.Add(i); newInfos.Add(info); } } } } walkedStates.Remove(state); } void ParsingTable::LookAheadInfo::Walk(Ptr table, Ptr previous, vint state, collections::List>& newInfos) { SortedList walkedStates; WalkInternal(table, previous, state, walkedStates, newInfos); } /*********************************************************************** ParsingTable::TransitionItem ***********************************************************************/ enum TransitionLevel { ReduceTransition, LeftRecursiveReduceTransition, NormalTransition, }; TransitionLevel GetTransitionLevel(Ptr t) { bool hasReduce=false; bool hasLrReduce=false; for (auto ins : t->instructions) { switch(ins.instructionType) { case ParsingTable::Instruction::Reduce: hasReduce=true; break; case ParsingTable::Instruction::LeftRecursiveReduce: hasLrReduce=true; break; default:; } } return hasLrReduce?LeftRecursiveReduceTransition: hasReduce?ReduceTransition: NormalTransition; } ParsingTable::TransitionItem::OrderResult ParsingTable::TransitionItem::CheckOrder(Ptr t1, Ptr t2, OrderResult defaultResult) { if(t1->token!=t2->token) return UnknownOrder; if (defaultResult != UnknownOrder) { TransitionLevel level1 = GetTransitionLevel(t1); TransitionLevel level2 = GetTransitionLevel(t2); if (level1 > level2) return CorrectOrder; if (level1 < level2) return WrongOrder; } vint ic1=t1->stackPattern.Count(); vint ic2=t2->stackPattern.Count(); vint ic=ic1stackPattern[i]; vint s2=t2->stackPattern[i]; if(s1>s2) { return CorrectOrder; } else if(s1token==TokenFinish) { if(ic1>ic2) { return CorrectOrder; } else if(ic1 t1, Ptr t2, OrderResult defaultResult) { OrderResult order=CheckOrder(t1, t2, defaultResult); switch(order) { case CorrectOrder: return -1; case WrongOrder: return 1; default: return 0; } } template void ParsingTable::IO(TIO& io) { io << ambiguity << attributeInfos << treeTypeInfos << treeFieldInfos << tokenCount << stateCount << tokenInfos << discardTokenInfos << stateInfos << ruleInfos << transitionBags ; } ParsingTable::ParsingTable(stream::IStream& input) { stream::internal::ContextFreeReader reader(input); IO(reader); } void ParsingTable::Serialize(stream::IStream& output) { stream::internal::ContextFreeWriter writer(output); IO(writer); } /*********************************************************************** ParsingTable ***********************************************************************/ #if defined(VCZH_GCC) && defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wnull-dereference" #endif ParsingTable::ParsingTable(vint _attributeInfoCount, vint _treeTypeInfoCount, vint _treeFieldInfoCount, vint _tokenCount, vint discardTokenCount, vint _stateCount, vint _ruleCount) :ambiguity(false) ,tokenCount(_tokenCount+UserTokenStart) ,stateCount(_stateCount) ,attributeInfos(_attributeInfoCount) ,treeTypeInfos(_treeTypeInfoCount) ,treeFieldInfos(_treeFieldInfoCount) ,tokenInfos(_tokenCount+UserTokenStart) ,discardTokenInfos(discardTokenCount) ,stateInfos(_stateCount) ,ruleInfos(_ruleCount) ,transitionBags((_tokenCount+UserTokenStart)*_stateCount) { } ParsingTable::~ParsingTable() { } bool ParsingTable::GetAmbiguity() { return ambiguity; } void ParsingTable::SetAmbiguity(bool value) { ambiguity=value; } vint ParsingTable::GetAttributeInfoCount() { return attributeInfos.Count(); } Ptr ParsingTable::GetAttributeInfo(vint index) { return attributeInfos[index]; } void ParsingTable::SetAttributeInfo(vint index, Ptr info) { attributeInfos[index]=info; } vint ParsingTable::GetTreeTypeInfoCount() { return treeTypeInfos.Count(); } const ParsingTable::TreeTypeInfo& ParsingTable::GetTreeTypeInfo(vint index) { return treeTypeInfos[index]; } const ParsingTable::TreeTypeInfo& ParsingTable::GetTreeTypeInfo(const WString& type) { vint index=treeTypeInfoMap.Keys().IndexOf(type); if(index==-1) return *(const TreeTypeInfo*)0; return treeTypeInfos[treeTypeInfoMap.Values().Get(index)]; } void ParsingTable::SetTreeTypeInfo(vint index, const TreeTypeInfo& info) { treeTypeInfos[index]=info; } vint ParsingTable::GetTreeFieldInfoCount() { return treeFieldInfos.Count(); } const ParsingTable::TreeFieldInfo& ParsingTable::GetTreeFieldInfo(vint index) { return treeFieldInfos[index]; } const ParsingTable::TreeFieldInfo& ParsingTable::GetTreeFieldInfo(const WString& type, const WString& field) { Pair key(type, field); vint index=treeFieldInfoMap.Keys().IndexOf(key); if(index==-1) return *(const TreeFieldInfo*)0; return treeFieldInfos[treeFieldInfoMap.Values().Get(index)]; } void ParsingTable::SetTreeFieldInfo(vint index, const TreeFieldInfo& info) { treeFieldInfos[index]=info; } vint ParsingTable::GetTokenCount() { return tokenCount; } const ParsingTable::TokenInfo& ParsingTable::GetTokenInfo(vint token) { return tokenInfos[token]; } void ParsingTable::SetTokenInfo(vint token, const TokenInfo& info) { tokenInfos[token]=info; } vint ParsingTable::GetDiscardTokenCount() { return discardTokenInfos.Count(); } const ParsingTable::TokenInfo& ParsingTable::GetDiscardTokenInfo(vint token) { return discardTokenInfos[token]; } void ParsingTable::SetDiscardTokenInfo(vint token, const TokenInfo& info) { discardTokenInfos[token]=info; } vint ParsingTable::GetStateCount() { return stateCount; } const ParsingTable::StateInfo& ParsingTable::GetStateInfo(vint state) { return stateInfos[state]; } void ParsingTable::SetStateInfo(vint state, const StateInfo& info) { stateInfos[state]=info; } vint ParsingTable::GetRuleCount() { return ruleInfos.Count(); } const ParsingTable::RuleInfo& ParsingTable::GetRuleInfo(const WString& ruleName) { vint index=ruleMap.Keys().IndexOf(ruleName); if(index==-1) return *(const RuleInfo*)0; return ruleInfos[ruleMap.Values().Get(index)]; } const ParsingTable::RuleInfo& ParsingTable::GetRuleInfo(vint rule) { return ruleInfos[rule]; } void ParsingTable::SetRuleInfo(vint rule, const RuleInfo& info) { ruleInfos[rule]=info; } const regex::RegexLexer& ParsingTable::GetLexer() { return *lexer.Obj(); } Ptr ParsingTable::GetTransitionBag(vint state, vint token) { return transitionBags[state*tokenCount+token]; } void ParsingTable::SetTransitionBag(vint state, vint token, Ptr bag) { transitionBags[state*tokenCount+token]=bag; } void ParsingTable::Initialize() { List tokens; for (auto info : From(tokenInfos).Skip(UserTokenStart)) { tokens.Add(info.regex); } for (auto info : discardTokenInfos) { tokens.Add(info.regex); } vint regexTokenIndex = 0; for (vint i = UserTokenStart; i < tokenInfos.Count(); i++) { tokenInfos[i].regexTokenIndex = regexTokenIndex++; } for (vint i = 0; i < discardTokenInfos.Count(); i++) { discardTokenInfos[i].regexTokenIndex = regexTokenIndex++; } lexer = Ptr(new RegexLexer(tokens)); ruleMap.Clear(); for (auto [rule, index] : indexed(ruleInfos)) { ruleMap.Add(rule.name, index); } for (vint i = 0; i < stateInfos.Count(); i++) { StateInfo& info = stateInfos[i]; info.ruleAmbiguousType = ruleInfos[ruleMap[info.ruleName]].ambiguousType; } treeTypeInfoMap.Clear(); for (auto [info, index] : indexed(treeTypeInfos)) { treeTypeInfoMap.Add(info.type, index); } treeFieldInfoMap.Clear(); for (auto [info, index] : indexed(treeFieldInfos)) { Pair key(info.type, info.field); treeFieldInfoMap.Add(key, index); } } bool ParsingTable::IsInputToken(vint regexTokenIndex) { return regexTokenIndex>=0 && regexTokenIndex=tokenCount-UserTokenStart?regexTokenIndex-(tokenCount-UserTokenStart):-1; } #if defined(VCZH_GCC) && defined(__clang__) #pragma clang diagnostic pop #endif } } } /*********************************************************************** .\PARSINGTREE.CPP ***********************************************************************/ /*********************************************************************** Author: Zihan Chen (vczh) Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ #if defined(VCZH_GCC) && defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wnull-dereference" #endif namespace vl { using namespace collections; namespace parsing { /*********************************************************************** ParsingTreeNode::TraversalVisitor ***********************************************************************/ ParsingTreeNode::TraversalVisitor::TraversalVisitor(TraverseDirection _direction) :direction(_direction) { } void ParsingTreeNode::TraversalVisitor::BeforeVisit(ParsingTreeToken* node) { } void ParsingTreeNode::TraversalVisitor::AfterVisit(ParsingTreeToken* node) { } void ParsingTreeNode::TraversalVisitor::BeforeVisit(ParsingTreeObject* node) { } void ParsingTreeNode::TraversalVisitor::AfterVisit(ParsingTreeObject* node) { } void ParsingTreeNode::TraversalVisitor::BeforeVisit(ParsingTreeArray* node) { } void ParsingTreeNode::TraversalVisitor::AfterVisit(ParsingTreeArray* node) { } void ParsingTreeNode::TraversalVisitor::Visit(ParsingTreeToken* node) { BeforeVisit(node); AfterVisit(node); } void ParsingTreeNode::TraversalVisitor::Visit(ParsingTreeObject* node) { BeforeVisit(node); switch(direction) { case TraverseDirection::ByTextPosition: { for (auto node : node->GetSubNodes()) { node->Accept(this); } } break; case TraverseDirection::ByStorePosition: { for (auto node : node->GetMembers().Values()) { node->Accept(this); } } break; } AfterVisit(node); } void ParsingTreeNode::TraversalVisitor::Visit(ParsingTreeArray* node) { BeforeVisit(node); switch(direction) { case TraverseDirection::ByTextPosition: { for (auto node : node->GetSubNodes()) { node->Accept(this); } } break; case TraverseDirection::ByStorePosition: { for (auto node : node->GetItems()) { node->Accept(this); } } break; } AfterVisit(node); } /*********************************************************************** ParsingTreeNode ***********************************************************************/ bool ParsingTreeNode::BeforeAddChild(Ptr node) { return node->parent==0; } void ParsingTreeNode::AfterAddChild(Ptr node) { node->parent=this; ClearQueryCache(); } bool ParsingTreeNode::BeforeRemoveChild(Ptr node) { return node->parent!=0; } void ParsingTreeNode::AfterRemoveChild(Ptr node) { node->parent=0; ClearQueryCache(); } ParsingTreeNode::ParsingTreeNode(const ParsingTextRange& _codeRange) :codeRange(_codeRange) ,parent(0) { } ParsingTreeNode::~ParsingTreeNode() { } ParsingTextRange ParsingTreeNode::GetCodeRange() { return codeRange; } void ParsingTreeNode::SetCodeRange(const ParsingTextRange& range) { codeRange=range; } void ParsingTreeNode::InitializeQueryCache() { const NodeList& subNodes=GetSubNodesInternal(); ClearQueryCache(); auto subNodesExists = &subNodes; if(subNodesExists) { for (auto node : subNodes) { node->InitializeQueryCache(); } //if (codeRange.start.IsInvalid() || codeRange.start.IsInvalid()) { for (auto subNode : subNodes) { const auto& subRange = subNode->codeRange; const auto& min = !subRange.start.IsInvalid() ? subRange.start : subRange.end; const auto& max = !subRange.end.IsInvalid() ? subRange.end : subRange.start; if (codeRange.start.IsInvalid() || (!min.IsInvalid() && codeRange.start > min)) { codeRange.start = min; } if (codeRange.end.IsInvalid() || (!max.IsInvalid() && codeRange.end < max)) { codeRange.end = max; } } } CopyFrom( cachedOrderedSubNodes, From(subNodes) .Where([=](auto&& node) { const auto& range = node->GetCodeRange(); return !range.start.IsInvalid() && !range.end.IsInvalid(); }) .OrderByKey([](auto&& node) { return node->GetCodeRange().start; }) ); } } void ParsingTreeNode::ClearQueryCache() { cachedOrderedSubNodes.Clear(); } ParsingTreeNode* ParsingTreeNode::GetParent() { return parent; } const ParsingTreeNode::NodeList& ParsingTreeNode::GetSubNodes() { return cachedOrderedSubNodes; } ParsingTreeNode* ParsingTreeNode::FindSubNode(const ParsingTextPos& position) { return FindSubNode(ParsingTextRange(position, position)); } ParsingTreeNode* ParsingTreeNode::FindSubNode(const ParsingTextRange& range) { if (codeRange.start <= range.start && range.end <= codeRange.end) { vint start = 0; vint end = cachedOrderedSubNodes.Count() - 1; while (start <= end) { vint selected = (start + end) / 2; ParsingTreeNode* selectedNode = cachedOrderedSubNodes[selected].Obj(); const ParsingTextRange& selectedRange = selectedNode->codeRange; if (range.endselectedRange.end) { start = selected + 1; } else if (selectedRange.start <= range.start && range.end <= selectedRange.end) { return selectedNode; } else { return this; } } } return this; } ParsingTreeNode* ParsingTreeNode::FindDeepestNode(const ParsingTextPos& position) { return FindDeepestNode(ParsingTextRange(position, position)); } ParsingTreeNode* ParsingTreeNode::FindDeepestNode(const ParsingTextRange& range) { ParsingTreeNode* result=0; ParsingTreeNode* node=this; do { result=node; node=node->FindSubNode(range); }while(result!=node); return result; } /*********************************************************************** ParsingTreeToken ***********************************************************************/ const ParsingTreeToken::NodeList& ParsingTreeToken::GetSubNodesInternal() { return *(NodeList*)0; } ParsingTreeToken::ParsingTreeToken(const WString& _value, vint _tokenIndex, const ParsingTextRange& _codeRange) :ParsingTreeNode(_codeRange) ,value(_value) ,tokenIndex(_tokenIndex) { } ParsingTreeToken::~ParsingTreeToken() { } void ParsingTreeToken::Accept(IVisitor* visitor) { visitor->Visit(this); } Ptr ParsingTreeToken::Clone() { auto clone = Ptr(new ParsingTreeToken(value, tokenIndex, codeRange)); return clone; } vint ParsingTreeToken::GetTokenIndex() { return tokenIndex; } void ParsingTreeToken::SetTokenIndex(vint _tokenIndex) { tokenIndex=_tokenIndex; } const WString& ParsingTreeToken::GetValue() { return value; } void ParsingTreeToken::SetValue(const WString& _value) { value=_value; } /*********************************************************************** ParsingTreeObject ***********************************************************************/ const ParsingTreeObject::NodeList& ParsingTreeObject::GetSubNodesInternal() { return members.Values(); } ParsingTreeObject::ParsingTreeObject(const WString& _type, const ParsingTextRange& _codeRange) :ParsingTreeNode(_codeRange) ,type(_type) { } ParsingTreeObject::~ParsingTreeObject() { } void ParsingTreeObject::Accept(IVisitor* visitor) { visitor->Visit(this); } Ptr ParsingTreeObject::Clone() { auto clone = Ptr(new ParsingTreeObject(type, codeRange)); CopyFrom(clone->rules, rules); for(vint i=0;i node=members.Values().Get(i)->Clone(); clone->SetMember(name, node); } return clone; } const WString& ParsingTreeObject::GetType() { return type; } void ParsingTreeObject::SetType(const WString& _type) { type=_type; } ParsingTreeObject::NodeMap& ParsingTreeObject::GetMembers() { return members; } Ptr ParsingTreeObject::GetMember(const WString& name) { vint index=members.Keys().IndexOf(name); if(index==-1) return 0; return members.Values().Get(index); } bool ParsingTreeObject::SetMember(const WString& name, Ptr node) { vint index=members.Keys().IndexOf(name); if(index!=-1) { Ptr previous=members.Values().Get(index); if(previous==node) return true; if(!BeforeRemoveChild(previous) || !BeforeAddChild(node)) return false; members.Remove(name); AfterRemoveChild(previous); } members.Add(name, node); AfterAddChild(node); return true; } bool ParsingTreeObject::RemoveMember(const WString& name) { vint index=members.Keys().IndexOf(name); if(index!=-1) { Ptr previous=members.Values().Get(index); if(BeforeRemoveChild(previous)) { members.Remove(name); AfterRemoveChild(previous); return true; } } return false; } const ParsingTreeObject::NameList& ParsingTreeObject::GetMemberNames() { return members.Keys(); } ParsingTreeObject::RuleList& ParsingTreeObject::GetCreatorRules() { return rules; } /*********************************************************************** ParsingTreeArray ***********************************************************************/ const ParsingTreeArray::NodeList& ParsingTreeArray::GetSubNodesInternal() { return items; } ParsingTreeArray::ParsingTreeArray(const WString& _elementType, const ParsingTextRange& _codeRange) :ParsingTreeNode(_codeRange) ,elementType(_elementType) { } ParsingTreeArray::~ParsingTreeArray() { } void ParsingTreeArray::Accept(IVisitor* visitor) { visitor->Visit(this); } Ptr ParsingTreeArray::Clone() { auto clone = Ptr(new ParsingTreeArray(elementType, codeRange)); for(vint i=0;i node=items.Get(i)->Clone(); clone->AddItem(node); } return clone; } const WString& ParsingTreeArray::GetElementType() { return elementType; } void ParsingTreeArray::SetElementType(const WString& _elementType) { elementType=_elementType; } ParsingTreeArray::NodeArray& ParsingTreeArray::GetItems() { return items; } Ptr ParsingTreeArray::GetItem(vint index) { if(0<=index && index node) { if(0<=index && index node) { return InsertItem(items.Count(), node); } bool ParsingTreeArray::InsertItem(vint index, Ptr node) { if(0<=index && index<=items.Count()) { if(BeforeAddChild(node)) { items.Insert(index, node); AfterAddChild(node); return true; } } return false; } bool ParsingTreeArray::RemoveItem(vint index) { if(0<=index && index previous=items[index]; if(BeforeRemoveChild(previous)) { items.RemoveAt(index); AfterRemoveChild(previous); return true; } } return false; } bool ParsingTreeArray::RemoveItem(ParsingTreeNode* node) { return RemoveItem(items.IndexOf(node)); } vint ParsingTreeArray::IndexOfItem(ParsingTreeNode* node) { return items.IndexOf(node); } bool ParsingTreeArray::ContainsItem(ParsingTreeNode* node) { return items.Contains(node); } vint ParsingTreeArray::Count() { return items.Count(); } bool ParsingTreeArray::Clear() { for (auto node : items) { if(!BeforeRemoveChild(node)) return false; } for (auto node : items) { AfterRemoveChild(node); } items.Clear(); return true; } /*********************************************************************** ParsingError ***********************************************************************/ ParsingError::ParsingError() :token(0) ,parsingTree(0) { } ParsingError::ParsingError(const WString& _errorMessage) :token(0) ,parsingTree(0) ,errorMessage(_errorMessage) { } ParsingError::ParsingError(const regex::RegexToken* _token, const WString& _errorMessage) :token(_token) ,parsingTree(0) ,errorMessage(_errorMessage) { if(token) { codeRange.start.row=_token->rowStart; codeRange.start.column=_token->columnStart; codeRange.start.index=_token->start; codeRange.end.row=_token->rowEnd; codeRange.end.column=_token->columnEnd; codeRange.end.index=_token->start+_token->length-1; codeRange.codeIndex = _token->codeIndex; } } ParsingError::ParsingError(ParsingTreeCustomBase* _parsingTree, const WString& _errorMessage) :codeRange(_parsingTree->codeRange) ,token(0) ,parsingTree(_parsingTree) ,errorMessage(_errorMessage) { } ParsingError::~ParsingError() { } /*********************************************************************** ParsingEmptyPrintNodeRecorder ***********************************************************************/ ParsingEmptyPrintNodeRecorder::ParsingEmptyPrintNodeRecorder() { } ParsingEmptyPrintNodeRecorder::~ParsingEmptyPrintNodeRecorder() { } void ParsingEmptyPrintNodeRecorder::Record(ParsingTreeCustomBase* node, const ParsingTextRange& range) { } /*********************************************************************** ParsingMultiplePrintNodeRecorder ***********************************************************************/ ParsingMultiplePrintNodeRecorder::ParsingMultiplePrintNodeRecorder() { } ParsingMultiplePrintNodeRecorder::~ParsingMultiplePrintNodeRecorder() { } void ParsingMultiplePrintNodeRecorder::AddRecorder(Ptr recorder) { recorders.Add(recorder); } void ParsingMultiplePrintNodeRecorder::Record(ParsingTreeCustomBase* node, const ParsingTextRange& range) { for (auto recorder : recorders) { recorder->Record(node, range); } } /*********************************************************************** ParsingOriginalLocationRecorder ***********************************************************************/ ParsingOriginalLocationRecorder::ParsingOriginalLocationRecorder(Ptr _recorder) :recorder(_recorder) { } ParsingOriginalLocationRecorder::~ParsingOriginalLocationRecorder() { } void ParsingOriginalLocationRecorder::Record(ParsingTreeCustomBase* node, const ParsingTextRange& range) { auto codeRange = node->codeRange; codeRange.codeIndex = range.codeIndex; recorder->Record(node, codeRange); } /*********************************************************************** ParsingGeneratedLocationRecorder ***********************************************************************/ ParsingGeneratedLocationRecorder::ParsingGeneratedLocationRecorder(RangeMap& _rangeMap) :rangeMap(_rangeMap) { } ParsingGeneratedLocationRecorder::~ParsingGeneratedLocationRecorder() { } void ParsingGeneratedLocationRecorder::Record(ParsingTreeCustomBase* node, const ParsingTextRange& range) { rangeMap.Add(node, range); } /*********************************************************************** ParsingUpdateLocationRecorder ***********************************************************************/ ParsingUpdateLocationRecorder::ParsingUpdateLocationRecorder() { } ParsingUpdateLocationRecorder::~ParsingUpdateLocationRecorder() { } void ParsingUpdateLocationRecorder::Record(ParsingTreeCustomBase* node, const ParsingTextRange& range) { node->codeRange = range; } /*********************************************************************** ParsingWriter ***********************************************************************/ void ParsingWriter::HandleChar(wchar_t c) { lastPos = currentPos; switch (c) { case L'\n': currentPos.index++; currentPos.row++; currentPos.column = 0; break; default: currentPos.index++; currentPos.column++; } } ParsingWriter::ParsingWriter(stream::TextWriter& _writer, Ptr _recorder, vint _codeIndex) :writer(_writer) , recorder(_recorder) , codeIndex(_codeIndex) , lastPos(-1, 0, -1) , currentPos(0, 0, 0) { } ParsingWriter::~ParsingWriter() { } void ParsingWriter::WriteChar(wchar_t c) { writer.WriteChar(c); if (!recorder) return; HandleChar(c); } void ParsingWriter::WriteString(const wchar_t* string, vint charCount) { writer.WriteString(string, charCount); if (!recorder) return; for (vint i = 0; i < charCount; i++) { HandleChar(string[i]); } } void ParsingWriter::BeforePrint(ParsingTreeCustomBase* node) { if (!recorder) return; nodePositions.Add(NodePosPair(node, currentPos)); } void ParsingWriter::AfterPrint(ParsingTreeCustomBase* node) { if (!recorder) return; auto pair = nodePositions[nodePositions.Count() - 1]; nodePositions.RemoveAt(nodePositions.Count() - 1); CHECK_ERROR(pair.key == node, L"vl::parsing::ParsingWriter::AfterPrint(ParsingTreeNode*)#BeforePrint and AfterPrint should be call in pairs."); ParsingTextRange range(pair.value, lastPos, codeIndex); recorder->Record(node, range); } } } #if defined(VCZH_GCC) && defined(__clang__) #pragma clang diagnostic pop #endif /*********************************************************************** .\JSON\PARSINGJSON.CPP ***********************************************************************/ namespace vl { namespace parsing { namespace json { using namespace stream; using namespace collections; /*********************************************************************** Unescaping Function Foward Declarations ***********************************************************************/ void JsonUnescapingString(vl::parsing::ParsingToken& value, const vl::collections::List& tokens) { value.value = GenerateToStream([&](StreamWriter& writer) { JsonUnescapeString(value.value.Sub(1, value.value.Length() - 2), writer); }); } /*********************************************************************** JsonPrintVisitor ***********************************************************************/ class JsonPrintVisitor : public Object, public JsonNode::IVisitor { public: TextWriter& writer; JsonPrintVisitor(TextWriter& _writer) :writer(_writer) { } void Visit(JsonLiteral* node) { switch(node->value) { case JsonLiteral::JsonValue::True: writer.WriteString(L"true"); break; case JsonLiteral::JsonValue::False: writer.WriteString(L"false"); break; case JsonLiteral::JsonValue::Null: writer.WriteString(L"null"); break; } } void Visit(JsonString* node) { writer.WriteChar(L'\"'); JsonEscapeString(node->content.value, writer); writer.WriteChar(L'\"'); } void Visit(JsonNumber* node) { writer.WriteString(node->content.value); } void Visit(JsonArray* node) { writer.WriteChar(L'['); for (auto [item, i] : indexed(node->items)) { if(i>0) writer.WriteChar(L','); item->Accept(this); } writer.WriteChar(L']'); } void Visit(JsonObjectField* node) { writer.WriteChar(L'\"'); JsonEscapeString(node->name.value, writer); writer.WriteString(L"\":"); node->value->Accept(this); } void Visit(JsonObject* node) { writer.WriteChar(L'{'); for (auto [field, i] : indexed(node->fields)) { if(i>0) writer.WriteChar(L','); field->Accept(this); } writer.WriteChar(L'}'); } }; /*********************************************************************** API ***********************************************************************/ void JsonEscapeString(const WString& text, stream::TextWriter& writer) { const wchar_t* reading=text.Buffer(); while(wchar_t c=*reading++) { switch(c) { case L'\"': writer.WriteString(L"\\\""); break; case L'\\': writer.WriteString(L"\\\\"); break; case L'/': writer.WriteString(L"\\/"); break; case L'\b': writer.WriteString(L"\\b"); break; case L'\f': writer.WriteString(L"\\f"); break; case L'\n': writer.WriteString(L"\\n"); break; case L'\r': writer.WriteString(L"\\r"); break; case L'\t': writer.WriteString(L"\\t"); break; default: writer.WriteChar(c); } } } vuint16_t GetHex(wchar_t c) { if(L'0'<=c && c<=L'9') { return c-L'0'; } else if(L'A'<=c && c<=L'F') { return c-L'A'; } else if(L'a'<=c && c<=L'f') { return c-L'a'; } else { return 0; } } void JsonUnescapeString(const WString& text, stream::TextWriter& writer) { const wchar_t* reading=text.Buffer(); while(wchar_t c=*reading++) { if(c==L'\\' && *reading) { switch(c=*reading++) { case L'b': writer.WriteChar(L'\b'); break; case L'f': writer.WriteChar(L'\f'); break; case L'n': writer.WriteChar(L'\n'); break; case L'r': writer.WriteChar(L'\r'); break; case L't': writer.WriteChar(L'\t'); break; case L'u': { wchar_t h1, h2, h3, h4; if((h1=reading[0]) && (h2=reading[1]) && (h3=reading[2]) && (h4=reading[3])) { reading+=4; wchar_t h=(wchar_t)(vuint16_t)( (GetHex(h1)<<12) + (GetHex(h2)<<8) + (GetHex(h3)<<4) + (GetHex(h4)<<0) ); writer.WriteChar(h); } } break; default: writer.WriteChar(c); } } else { writer.WriteChar(c); } } } void JsonPrint(Ptr node, stream::TextWriter& writer) { JsonPrintVisitor visitor(writer); node->Accept(&visitor); } WString JsonToString(Ptr node) { return GenerateToStream([&](StreamWriter& writer) { JsonPrint(node, writer); }); } } } } /*********************************************************************** .\JSON\PARSINGJSON_AST.CPP ***********************************************************************/ /*********************************************************************** This file is generated by: Vczh Parser Generator From parser definition:ParsingJson.parser.txt Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { namespace json { /*********************************************************************** Visitor Pattern Implementation ***********************************************************************/ void JsonLiteral::Accept(JsonNode::IVisitor* visitor) { visitor->Visit(this); } void JsonString::Accept(JsonNode::IVisitor* visitor) { visitor->Visit(this); } void JsonNumber::Accept(JsonNode::IVisitor* visitor) { visitor->Visit(this); } void JsonArray::Accept(JsonNode::IVisitor* visitor) { visitor->Visit(this); } void JsonObjectField::Accept(JsonNode::IVisitor* visitor) { visitor->Visit(this); } void JsonObject::Accept(JsonNode::IVisitor* visitor) { visitor->Visit(this); } } } } namespace vl { namespace reflection { namespace description { #ifndef VCZH_DEBUG_NO_REFLECTION using namespace vl::parsing::json; #define PARSING_TOKEN_FIELD(NAME)\ CLASS_MEMBER_EXTERNALMETHOD_TEMPLATE(get_##NAME, NO_PARAMETER, vl::WString(ClassType::*)(), [](ClassType* node) { return node->NAME.value; }, L"*", L"*")\ CLASS_MEMBER_EXTERNALMETHOD_TEMPLATE(set_##NAME, { L"value" }, void(ClassType::*)(const vl::WString&), [](ClassType* node, const vl::WString& value) { node->NAME.value = value; }, L"*", L"*")\ CLASS_MEMBER_PROPERTY_REFERENCETEMPLATE(NAME, get_##NAME, set_##NAME, L"$This->$Name.value")\ IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonNode, system::JsonNode) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonLiteral, system::JsonLiteral) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonLiteral::JsonValue, system::JsonLiteral::JsonValue) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonString, system::JsonString) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonNumber, system::JsonNumber) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonArray, system::JsonArray) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonObjectField, system::JsonObjectField) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonObject, system::JsonObject) IMPL_TYPE_INFO_RENAME(vl::parsing::json::JsonNode::IVisitor, system::JsonNode::IVisitor) #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA BEGIN_CLASS_MEMBER(JsonNode) CLASS_MEMBER_METHOD_OVERLOAD(Accept, {L"visitor"}, void(JsonNode::*)(JsonNode::IVisitor* visitor)) END_CLASS_MEMBER(JsonNode) BEGIN_CLASS_MEMBER(JsonLiteral) CLASS_MEMBER_BASE(JsonNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) CLASS_MEMBER_FIELD(value) END_CLASS_MEMBER(JsonLiteral) BEGIN_ENUM_ITEM(JsonLiteral::JsonValue) ENUM_ITEM_NAMESPACE(JsonLiteral::JsonValue) ENUM_NAMESPACE_ITEM(True) ENUM_NAMESPACE_ITEM(False) ENUM_NAMESPACE_ITEM(Null) END_ENUM_ITEM(JsonLiteral::JsonValue) BEGIN_CLASS_MEMBER(JsonString) CLASS_MEMBER_BASE(JsonNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(content) END_CLASS_MEMBER(JsonString) BEGIN_CLASS_MEMBER(JsonNumber) CLASS_MEMBER_BASE(JsonNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(content) END_CLASS_MEMBER(JsonNumber) BEGIN_CLASS_MEMBER(JsonArray) CLASS_MEMBER_BASE(JsonNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) CLASS_MEMBER_FIELD(items) END_CLASS_MEMBER(JsonArray) BEGIN_CLASS_MEMBER(JsonObjectField) CLASS_MEMBER_BASE(JsonNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(name) CLASS_MEMBER_FIELD(value) END_CLASS_MEMBER(JsonObjectField) BEGIN_CLASS_MEMBER(JsonObject) CLASS_MEMBER_BASE(JsonNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) CLASS_MEMBER_FIELD(fields) END_CLASS_MEMBER(JsonObject) BEGIN_INTERFACE_MEMBER(JsonNode::IVisitor) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(JsonNode::IVisitor::*)(JsonLiteral* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(JsonNode::IVisitor::*)(JsonString* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(JsonNode::IVisitor::*)(JsonNumber* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(JsonNode::IVisitor::*)(JsonArray* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(JsonNode::IVisitor::*)(JsonObjectField* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(JsonNode::IVisitor::*)(JsonObject* node)) END_INTERFACE_MEMBER(JsonNode) #endif #undef PARSING_TOKEN_FIELD #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA class JsonTypeLoader : public vl::Object, public ITypeLoader { public: void Load(ITypeManager* manager) { ADD_TYPE_INFO(vl::parsing::json::JsonNode) ADD_TYPE_INFO(vl::parsing::json::JsonLiteral) ADD_TYPE_INFO(vl::parsing::json::JsonLiteral::JsonValue) ADD_TYPE_INFO(vl::parsing::json::JsonString) ADD_TYPE_INFO(vl::parsing::json::JsonNumber) ADD_TYPE_INFO(vl::parsing::json::JsonArray) ADD_TYPE_INFO(vl::parsing::json::JsonObjectField) ADD_TYPE_INFO(vl::parsing::json::JsonObject) ADD_TYPE_INFO(vl::parsing::json::JsonNode::IVisitor) } void Unload(ITypeManager* manager) { } }; #endif #endif bool JsonLoadTypes() { #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA ITypeManager* manager = GetGlobalTypeManager(); if(manager) { auto loader = Ptr(new JsonTypeLoader); return manager->AddTypeLoader(loader); } #endif return false; } } } } /*********************************************************************** .\JSON\PARSINGJSON_PARSER.CPP ***********************************************************************/ /*********************************************************************** This file is generated by: Vczh Parser Generator From parser definition:ParsingJson.parser.txt Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { namespace json { /*********************************************************************** ParserText ***********************************************************************/ const wchar_t* const parserTextBuffer[] = { L"" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"// AST" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"" L"\r\n" , L"class Node\t\t\t\t\t\t\t\t@Document(\"Base class of JSON nodes.\")" L"\r\n" , L"{" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Literal:Node\t\t\t\t\t\t@Document(\"Literal value node that is not number or string.\")" L"\r\n" , L"{" L"\r\n" , L"\tenum Value\t\t\t\t\t\t\t@Document(\"Literal value.\")" L"\r\n" , L"\t{" L"\r\n" , L"\t\tTrue\t\t\t\t\t\t\t@Document(\"A boolean literal: true.\")," L"\r\n" , L"\t\tFalse\t\t\t\t\t\t\t@Document(\"A boolean literal: false.\")," L"\r\n" , L"\t\tNull\t\t\t\t\t\t\t@Document(\"A null literal.\")," L"\r\n" , L"\t}" L"\r\n" , L"" L"\r\n" , L"\tValue value\t\t\t\t\t\t\t@Document(\"The literal value.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class String:Node\t\t\t\t\t\t@Document(\"String literal value node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken content(JsonUnescapingString)\t@Document(\"Content of the string literal.\"), @Color(\"String\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Number:Node\t\t\t\t\t\t@Document(\"Number literal value node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken content\t\t\t\t\t\t@Document(\"Content of the number literal.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Array:Node\t\t\t\t\t\t@Document(\"Array node.\")" L"\r\n" , L"{" L"\r\n" , L"\tNode[] items\t\t\t\t\t\t@Document(\"Array elements.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class ObjectField:Node\t\t\t\t\t@Document(\"Object property node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken name(JsonUnescapingString)\t@Document(\"Property name.\"), @Color(\"AttName\");" L"\r\n" , L"\tNode value\t\t\t\t\t\t\t@Document(\"Property value.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Object:Node\t\t\t\t\t\t@Document(\"Object node.\")" L"\r\n" , L"{" L"\r\n" , L"\tObjectField[] fields\t\t\t\t@Document(\"Object properties.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"// Lexer" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"" L"\r\n" , L"token TRUEVALUE = \"true\"\t\t\t\t\t\t\t@Color(\"Keyword\");" L"\r\n" , L"token FALSEVALUE = \"false\"\t\t\t\t\t\t\t@Color(\"Keyword\");" L"\r\n" , L"token NULLVALUE = \"null\"\t\t\t\t\t\t\t@Color(\"Keyword\");" L"\r\n" , L"token OBJOPEN = \"\\{\"\t\t\t\t\t\t\t\t@Color(\"Boundary\");" L"\r\n" , L"token OBJCLOSE = \"\\}\"\t\t\t\t\t\t\t\t@Color(\"Boundary\");" L"\r\n" , L"token ARROPEN = \"\\[\"\t\t\t\t\t\t\t\t@Color(\"Boundary\");" L"\r\n" , L"token ARRCLOSE = \"\\]\"\t\t\t\t\t\t\t\t@Color(\"Boundary\");" L"\r\n" , L"token COMMA = \",\";" L"\r\n" , L"token COLON = \":\";" L"\r\n" , L"token NUMBER = \"[\\-]?\\d+(.\\d+)?([eE][+\\-]?\\d+)?\"\t@Color(\"Number\");" L"\r\n" , L"token STRING = \"\"\"([^\\\\\"\"]|\\\\[^u]|\\\\u\\d{4})*\"\"\"\t\t@ContextColor();" L"\r\n" , L"" L"\r\n" , L"discardtoken SPACE = \"\\s+\";" L"\r\n" , L"" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"// Rules" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"" L"\r\n" , L"rule Node JLiteral" L"\r\n" , L"\t= STRING:content as String" L"\r\n" , L"\t= NUMBER:content as Number" L"\r\n" , L"\t= \"true\" as Literal with {value = \"True\"}" L"\r\n" , L"\t= \"false\" as Literal with {value = \"False\"}" L"\r\n" , L"\t= \"null\" as Literal with {value = \"Null\"}" L"\r\n" , L"\t;" L"\r\n" , L"" L"\r\n" , L"rule ObjectField JField" L"\r\n" , L"\t= STRING:name \":\" JValue:value as ObjectField" L"\r\n" , L"\t;" L"\r\n" , L"" L"\r\n" , L"rule Object JObject" L"\r\n" , L"\t= \"{\" [JField:fields {\",\" JField:fields} ] \"}\" as Object" L"\r\n" , L"\t;" L"\r\n" , L"" L"\r\n" , L"rule Array JArray" L"\r\n" , L"\t= \"[\" [JValue:items {\",\" JValue:items} ] \"]\" as Array" L"\r\n" , L"\t;" L"\r\n" , L"" L"\r\n" , L"rule Node JValue" L"\r\n" , L"\t= !JLiteral" L"\r\n" , L"\t= !JObject" L"\r\n" , L"\t= !JArray" L"\r\n" , L"\t;" L"\r\n" , L"" L"\r\n" , L"rule Node JRoot" L"\r\n" , L"\t= !JObject" L"\r\n" , L"\t= !JArray" L"\r\n" , L"\t;" L"\r\n" }; const vint lengthTextBuffer[] = { 2, 68, 8, 68, 2, 58, 3, 3, 2, 87, 3, 47, 4, 53, 55, 44, 4, 2, 53, 3, 2, 64, 3, 101, 3, 2, 64, 3, 66, 3, 2, 48 , 3, 50, 3, 2, 63, 3, 83, 49, 3, 2, 50, 3, 59, 3, 2, 68, 10, 68, 2, 51, 53, 51, 49, 50, 49, 50, 20, 20, 68, 67, 2, 29 , 2, 68, 10, 68, 2, 20, 29, 29, 44, 46, 44, 4, 2, 25, 48, 4, 2, 21, 59, 4, 2, 19, 56, 4, 2, 18, 14, 13, 12, 4, 2, 17 , 13, 12, 4 }; const vint lengthTextBufferTotal = 2738; vl::WString JsonGetParserTextBuffer() { vl::collections::Array textBuffer(lengthTextBufferTotal + 1); wchar_t* reading = &textBuffer[0]; for(vint i = 0; i < sizeof(parserTextBuffer) / sizeof(*parserTextBuffer); i++) { memcpy(reading, parserTextBuffer[i], lengthTextBuffer[i] * sizeof(wchar_t)); reading += lengthTextBuffer[i]; } *reading = 0; return &textBuffer[0]; } /*********************************************************************** SerializedTable ***********************************************************************/ const vint parserBufferLength = 4281; // 19135 bytes before compressing const vint parserBufferBlock = 1024; const vint parserBufferRemain = 185; const vint parserBufferRows = 5; const char* const parserBuffer[] = { "\x00\x17\x00\x02\x83\x80\x07\x7D\x00\x82\x03\xFF\x48\x08\x82\x83\x86\x80\x22\x6F\x63\x35\x2D\x35\x36\x34\x3B\x83\x0C\x99\x0C\x80\x0A\x8A\x82\x21\x30\x73\x65\x20\x03\x3C\x30\x93\x38\x10\x6F\x66\x20\x0A\x23\x2F\x26\x26\x10\x6E\x6F\x24\x25\x33\x3E\x17\x7E\x01\x1B\x8A\x9B\x8E\x80\x00\x8A\x88\x8A\x16\x98\x88\x90\x1B\x8C\xA2\x80\x26\x69\x74\x25\x32\x31\x34\x34\x10\x3B\x4D\xF5\x22\x91\x93\x98\x10\x3A\x34\x61\x74\x20\x09\x37\x91\x9A\xAC\x37\x13\xE2\x0B\xA8\x92\x38\x13\x38\x3A\x72\x69\x2E\x27\x36\x9C\x9C\x82\x9E\x3E\x93\x95\x97\x8B\x8D\x8C\x84\xBB\x1F\xA1\xA3\x85\x93\x3F\x91\x95\x95\x2D\xAF\x94\xB4\x96\x98\x9C\x86\x9D\x0C\xF0\x91\x92\xB1\xA3\x8E\x0C\xC7\x03\xD3\x28\xAA\xB7\x30\x10\x37\xA4\x4B\xCD\x8F\xB1\xA3\xAA\x99\x33\xB6\x3B\x83\xB1\xA0\xA4\xB9\x8D\xBB\x81\x79\xA2\xA4\x86\x98\x92\x94\x96\x97\x30\x9E\xF5\x97\x99\x9E\xB7\x80\xC5\x3F\xF3\x82\xA4\x8F\xC4\x85\xDE\x00\x4E\x61\xA3\xB7\xCA\xA4\xA6\xA6\xA8\x6C\x52\xB1\xD3\x90\xD1\xC6\xD1\xC5\xA4\xBB\xC3\x96\xB4\x80\xBC\x90\xD5\x7C\xFE\xAE\xC1\xCB\xE4\xC0\xDB\xC3\xB6\x8D\x8F\xF9\xDD\xC4\x87\x05\x8D\xE7\x82\x81\x32\x3C\xA1\x3D\xEF\xE6\x08\xB8\xCC\xC5\xD7\xBF\xD2\x8F\xEB\x7B\xAC\xFF\xAF\xD2\xC2\xD9\xC3\xDA\xA1\x4F\x40\x76\x6E\x1B\x55\x05\x83\x40\x4F\x22\x1A\x19\x19\x63\x1A\x50\x1E\x1C\x6F\x30\x1B\x50\x1D\xEE\x43\x60\x7F\x5B\xE3\x73\x72\x76\x40\xD4\x53\x78\x68\x7E\xAB\x7D\x5D\x68\x60\xB0\x43\x63\x6E\x61\x08\x48\x61\x7D\x86\xD1\x5B\x4C\x02\x8B\x08\x8A\x8C\x83\x57\x9E\x57\x87\x6C\x8B\x04\xA6\x6A\x8C\x00\xA9\x79\x72\x8B\x7E\xDC\x66\x86\x4F\x86\x05\x61\x70\x02\x7C\x2D\x8C\x42\x07\x46\x4B\x82\x44\x14\x1A\x22\x44\x79\x67\x71\x9C\x7F\x70\x79\x80\x3D\x43\x85\x78\x42\x1E\x1B\x4D\x96\x40\x43\x2F\x16\x47\x5C\x23\x97\x52\x4B\x59\x69\x6B\x56\x64\x66\xC6\x76\x45\x03\x46\x6F\xA0\x9F\x18\x1B\x6F\x32\x14\x92\x01\x1B\x78\x92\x42\x64\x69\xA7\x16\x40\x80\xCE\x42\x84\x78\x7D\x04\x5F\x9C\x97\x40\x61\xA3\x96\x45\x99\x50\xB0\x42\x71\x59\x52\xAD\x9F\x9F\x95\x81\x99\x93\xA0\x91\x0F\x1B\x49\xA6\x7A\xEC\x61\x1E\x79\x19\x6C\x25\x14\x6B\x89\x80\xAB\x82\xA2\x86\x1B\x4E\x09\xAB\x40\x50\x10\x82\x86\x1C\x14\xB0\x41\x18\x45\x6E\xB0\x98\xA3\x9C\x75\x84\x97\x03\x46\xBB\xAA\x74\x1C\x1D\x4E\x33\xA5\x1B\xA4\x29\x86\x98\x91\x8E\x08\x5B\xA9\xB0\xAB\xAE\x93\x8E\x78\x72\x52\x43\x95\xA8\x86\xA7\x89\x98\x41\x93\xD7\xB1\x8B\x81\x83\x20\x0F\x81\x87\x84\x69\x1E\x74\x93\x3F\x71\x88\x45\xB9\x98\x74\xB6\x9C\xAF\xBA\x02\x4B\x15\x19\x1E\x77\x35\x94\x18\xB1\x04\x64\xB6\xAE\x9C\xE9\xBA\xAC\xB8\x00\xEE\xB0\xB2\xBC\xBD\x09\x65\xB4\x43\xB9\xB8\xAA\xBC\x41\xAF\x08\x48\xCD\xBD\xBC\x72\x33\xB3\xBA\xC0\x03\x44\xC9\xBC\x96\xD4\xBC\x8F\x19\x1D\x6E\x24\x11\x1A\x1C\x79\x00\xC2\x42\xBD\x0C\x52\xC9\xAC\xC5\x37\x9E\x47\xC5\xC6\x1B\xDD\xCF\xC3\xBD\x00\x28\xB3\xC9\xA5\x15\xC2\x17\xCA\xC6\x1C\xDE\xC5\x40\xC4\x02\x62\xC6\xC2\xA9\x3C\xF2\xC8\xC4\xCD\x2A\xF9\x48\xCC\x00\x3A\xF7\x99\x9F\x40\xC1\x6D\x12\x5A\x1C\x36\xFF\x3F\x88\x43\x4F\xED\xC2\x9A\x52\x78\x34\x1E\xCA\x1C\x1B\x40\x08\xC3\xC0\x2C\xEB\x7D\x78\x01\xFB\x84\x48\xC0\x52\xC5\x4D\x54\x90\xD8\x0C\x68\xD0\x72\x67\x59\xFA\x98\x42\xDB\x49\xCB\xD3\x03\xD1\x0C\x6E\xD9\x83\xB6\x74\x2E\xD4\x41\x7A\x08\x7C\xD7\xDF\x8C\x46\x21\xBC\x18\x19\x43\xFA\xD3\x42\xDB\x7C\x94\x68\x8A\xD8\x2C\xDC\xDC\xA5\x7B\x43\xE5\xB8\x65\x1B\x73\x0A\x01\xDB\x40\x63\xEC\x9D\x51\xE4\x03\x50\xB5\x1B\xD6\x87\xF4\xD0\x73\xA3\xA0\xC2\x48\xC3\x18\x53\xF3\x59\x02\xE8\x04\x76\xD2\x8D\x83\x86\xC2\x4E\xDA\x19\x82\xE4\x13\x1D\x03\xE8\x79\xE2\x43\xDF\x0D\x81\xE0\xA8\x19\x6A\xC4\x4A\xDA\x1B\xC2\xBC\xDB\xDE\xEE\x00\x3C\xE4\x1E\xEF\x83\xDC\xE2\x42\xE7\x51\xED\xE2\xE9\xE2\x6B\x65\xEA\xD7\x40\xA8\xCA\xA4\x1E\x91\xCA\x84\x44\x0A\xA6\x59\xE1\xF7\x63\x3F\xE4\xE5\xF6\xFB\xF9\xE2\xE3\xF7\xFB\xFA\xE5\xE9\xF9\x88\xFB\xF0\xEE\xF5\xBC\xFC\xEC\xEB\xEC\x42\xFD\x00\x14\x12\x15\x15\x45\x16\x11\x10\x13\xFC\x61\x7B\x40\x6A\x68\x2C\xA9\x19\x6A\x55\x0C\x36\x71\x02\x83\x26\x09\xFF\x53\x0D\x7F\xFF\x7C\x7E\xF3\x00\x06\x0D\xA6\x21\x20\x53\x1B\x38\x7C\x41\x78\x7E\x09\x2A\x0C\x0C\x09\xFE\x60\x81\x11\x02\x82\x21\xE1\x75\x0C\x0D\x6C\x0B\x73\x40\x10\x07\x72\xD3\x48\x42\x08\x4A\x0F\x08\x14\x45\x0E\x08\x15\x82\x20\x45\x5C\x1B\x0F\xDC\x0C\x21\x02\x92\x62\x3B\x77\x30\x83\x08\x13\x4F\x0F\x80", "\x1B\x80\x00\x87\x7D\x04\x84\x00\x58\x5C\x71\x83\x43\x21\x08\x52\x12\x0A\x0C\xB4\x86\x88\x1C\x9B\x0B\x68\x13\x1E\x87\x91\x4F\x81\x8A\x21\x84\x8D\x08\x55\x83\x20\x17\x5D\x01\x76\x00\x14\x05\x71\x11\x68\x53\x13\x4D\x0D\x09\x20\x04\x4A\x28\x2C\x16\x78\xFC\x13\x83\x09\x27\x03\x8E\x09\x6F\x83\x22\x0E\x46\x40\x7F\x32\xB0\x6D\x0A\x4D\x02\x09\x11\x52\x0F\x03\x8D\x05\x92\x20\x5B\x1C\x09\x0B\x5D\x0F\x06\x2E\x24\x0B\x05\x28\x0E\x06\x23\xAB\x09\x05\x1F\x28\x03\x0B\x65\x05\x09\x17\x5B\x0B\x04\x45\x8C\x93\x92\x95\x9A\x83\x40\x06\x45\x6F\xE9\x7A\x79\x09\x4E\x07\x0B\x06\x1B\x2B\x94\x81\x22\x07\x92\x5E\x1C\x08\x17\x22\x0D\x0A\x3E\x32\x93\x0B\x5E\x15\x0D\x2D\xB2\x95\x0E\x47\xBB\x0C\x06\x7D\x09\x06\x0A\x22\x0A\x80\x84\x16\x03\x23\x44\x53\x83\x14\x50\x01\x09\x21\x05\x0B\x87\x08\x30\x98\x17\x73\x0B\x04\xE3\x49\x24\x7D\xDF\x70\x65\xC9\x00\x0A\x08\xB2\x53\x4C\x89\x03\x38\x59\x37\x9A\x7C\x0C\x17\x12\x0F\x0D\x6F\x14\x0E\x64\x1B\x64\x0E\xDC\x4C\x26\x9D\x00\x00\x08\x08\x24\x0C\x07\x71\xA5\x6C\x0D\x3E\x06\x5B\x40\x76\x93\x4A\x04\x88\x26\x9F\xFC\x8D\x4F\x3A\xB0\x51\x7B\x81\x11\x6A\x9E\xF5\x9E\x9A\x71\x79\x98\x4B\x80\x8D\x2D\x98\x04\x25\x9A\x43\xE5\x91\x06\x7D\x1B\x26\xA2\x02\x29\xA0\x39\xBE\x0A\x06\x10\x13\x0F\x94\xA9\x80\x05\x47\xD9\x7A\x29\x45\x60\x26\xA3\x93\x2B\x2C\x10\x0D\x0A\x00\x8D\xB7\x9C\xA3\x20\x1D\x81\x20\x83\x92\xA5\x91\x29\x76\xA4\x27\x31\x6F\x52\x2B\xAD\xA5\x89\xB0\xA2\x04\x05\x85\x0E\x08\x20\x07\xA4\x85\x8D\x2A\x9E\x77\x09\x28\x1A\x20\x0B\x0F\xCD\x09\x3A\x24\x3D\x00\x06\x08\x54\x02\x0E\xA9\x03\xAD\x0F\x2C\xAE\xA7\x94\x3F\xA5\x82\x36\x21\x23\xA8\x45\xA4\x9C\x52\xCA\xAC\xA8\xA7\x8A\x39\xAA\x22\x06\x0A\x05\xC2\xA0\x04\xAC\xBD\xA6\xA8\xF8\x9D\xA2\x08\x60\x28\x84\xB0\xA7\x20\xAE\x63\xB7\x29\x59\x9E\x70\x04\xB4\x81\x3D\xAE\x6D\xBA\x9E\x40\x13\xAC\x84\x86\x05\xB4\x9B\x70\xA6\x9C\x3A\xF4\x05\x09\x8C\x70\x9C\x20\xF0\x94\x9E\x4F\xA0\x00\x08\x3F\xBB\x9E\x0A\x9C\x22\xB0\x37\x98\xBF\xA8\x17\x31\x05\x07\x1B\x20\xB5\x46\xCA\x0C\xB2\xA9\x30\xA1\x04\x13\xBC\xA5\x3D\xA5\xB5\x0D\xD3\xB1\x87\x75\x74\x0A\xB4\x69\x9C\x3E\xB4\xAF\x5E\x55\xB2\xA2\x6A\x08\xF3\x64\x0A\xB2\xDD\xB5\x76\x05\x33\x1A\x07\x46\x42\xB3\xB5\xDD\xB0\xA7\xA3\x52\x08\x97\x11\x35\xA4\x78\x8A\x12\xAA\x07\x43\xB3\xB7\x53\xB5\xAC\xAF\x93\x8A\x7B\xB7\xB8\xA5\x97\x60\xB6\xB9\x0F\xDE\x9C\xBE\x05\x32\x0C\x13\x46\x62\xB3\xB4\xEE\xBF\xA3\x0B\x43\xBB\x0B\x4A\x2C\xB5\xA7\xC9\x73\x0A\x9E\x4D\xA2\x04\x0B\x51\xBC\xBD\x8E\xAE\xBD\xAD\x20\x1D\x0A\x54\xDD\x06\xAE\x10\x1C\xB9\xAB\x4A\x06\xBE\x5C\xE8\xB0\x04\xF5\x92\xBA\x39\xF6\xAA\x28\xE5\x14\xB1\xBF\xF9\xAB\xBC\x33\xED\xAA\xC3\x49\x7D\x09\xBF\xFD\xBD\xBC\xBB\x96\xA3\xB0\x6B\xFE\x94\x20\x80\xAC\xB2\xB1\xE9\x8D\xB0\x19\x56\x92\x20\xBE\x52\xB4\xB5\x3E\x14\xB1\x86\xDC\x9B\xB6\x0E\xC3\x20\xA0\xBB\xA0\xC4\x63\x99\x64\xC4\x00\x26\xC5\x9E\xC6\xB4\xB2\xF5\x08\x6F\xB4\xBC\x62\x92\x20\xF8\x7D\xC7\x8C\x60\xB4\x13\x8D\x05\xCB\xB4\xCA\x7F\xA7\x1E\x69\xBB\x25\xDD\xB5\xA4\x76\xBF\x6F\xBD\x59\xF2\xB1\xBA\x27\xDD\xA0\xCA\x83\x73\xC1\x85\xA2\x0D\x0F\xFE\x8A\x78\xC0\x49\xD2\xAF\x92\x85\xCD\xC9\xDA\x4F\xCD\x76\x27\x20\x09\x83\xD5\xC6\xCD\x2B\xE8\xC8\xBF\xFA\xBC\xCA\x97\xB0\xBB\xC5\x00\xDD\x55\xBB\x0D\xBF\xBB\x0C\x45\x0B\x22\x3E\xE5\xBF\xCE\xE7\xAC\xCB\x81\xA2\x29\xC0\x51\x67\x23\xCD\x0F\xD3\xB9\xA1\x8B\xC4\xC3\x38\xFC\xBC\xA8\x27\x3C\xB9\x76\xEF\x6B\x9E\xEE\x9E\xBF\xCE\x2E\x14\x06\x9F\x88\x2B\xD3\x7A\x82\xC2\xAA\x04\xC6\xC1\x7D\xDB\x22\xC3\x29\xF3\xBA\x9E\x83\xD1\xC2\xA1\xF0\xC6\xC2\x48\xFE\xBF\xCE\x3B\xDB\x9E\xF2\x3F\xC0\x01\x20\xCA\x79\xD3\x49\x1B\x21\xAE\xC8\xC0\xB6\x25\xC2\xD6\xCA\x20\x18\xCA\xED\x26\xD2\x04\x54\xC0\xDA\xD8\x5A\xD1\xCD\x97\xAE\xDA\x79\x58\xC3\xB3\xD6\x19\x92\xD4\x6C\xC0\xBB\xD6\x84\x15\xDD\x9E\x61\xD2\xAB\x98\x86\xC7\xD9\x37\xC8\xD6\xB7\xBF\x67\xCD\x94\xAC\xD2\xCE\x66\xF0\xB3\xD2\xD2\x7B\xBB\x9D\xAE\xC2\x20\x80\x9F\xBF\x9C\x21\xD4\xC4\xF2\x12\xB6\xBC\xCA\x93\x82\x09\xF0\xD4\x0E\xB3\xDC\x99\xDE\xCF\xA5\x03\x23\xFF\xC3\xB5\xBE\xAE\xBA\x78\xD9\x83\xE5\xA3\xA8\xB2\xD3\xBE\xC2\xCD\x40\x5A\xF6\xD3\xDA\x31\x15\xD8\x41\x57\xDC\xE0\x3A\x3E\xD2\x9E\xDC\xCD\xCD\xB0\xDE\xD2\xAA\x07\xE0\xDB\x70\xE2\xD9\xC8\xB9\xCB\xD6\xBA\x3A\xF9\xB6\xD2\x79\xD8\xD1\x0C\x1B\xD4\x21\x4E", "\xF6\xCD\x3D\x81\xD8\xE1\xA2\x88\xC4\xD5\x55\xDF\xDB\xD4\xF7\xAD\xD1\xAB\xA7\xA8\xC3\xDC\xBB\xB5\xB7\x31\xD1\x04\x71\x88\x23\xE8\x1C\xF5\x77\xB8\x20\xAB\xB9\x47\xCD\xB5\x0C\x79\xA2\x00\xBA\x33\xE4\xD2\x59\xA2\x25\xE4\xBC\x7B\xB3\xE1\xCA\x7D\x7B\x40\x4A\x55\xE2\x19\xE2\xC5\xA0\x02\x31\x6A\x64\xC9\xC6\xDE\x96\x78\xDB\xB1\x8F\xA3\x20\x3C\xE7\xE9\x9C\xAF\xD9\x60\xA2\x03\x25\x9A\x64\x83\xEA\xC4\x9F\xF5\x70\xE8\xBF\xB2\x07\xD0\x84\x25\xE9\x8F\xE4\x08\xE9\xC9\xA1\xA7\xD2\xC2\x5F\xB9\x62\xD1\xEC\xBA\x10\xD5\xE8\xE0\x35\x75\xCF\xDD\x87\xB4\x20\x87\xBE\xEE\x8C\x83\xA4\x0F\x1A\xFC\x6A\x9E\x92\xA6\xB9\xE3\xB5\x70\xEC\x00\x11\x6A\xF2\x94\xF0\xEE\x40\x45\x98\xA0\xBF\xFE\x00\xEB\xB0\xBD\x84\x0F\xD2\xD8\x6F\xC9\xF4\x0C\x9D\x9D\xE7\xA3\x3C\xD8\xDD\xD6\xD4\xF8\x69\xED\x02\x2B\xEF\xAD\x94\xF6\xF3\x81\x25\xF1\xC9\x75\xDC\xBB\xE3\xAB\x85\xD2\x4C\xF9\xDC\xF7\xC8\x78\xF2\xA7\x80\xD1\xF9\x3B\xF1\xF5\xF8\xED\x2E\x05\xE8\x9B\x24\xF4\x10\x0B\xF8\xE6\x75\xCC\xB7\xF0\x82\x21\xF3\x0F\xC7\xFF\xF5\xB0\x5D\xF6\x40\x4A\xF3\xD0\x1F\x19\xFC\x33\xCF\xFA\xC1\x95\x53\xFB\x81\xF1\xD6\xFD\xFC\xF7\xD9\xDE\xD6\x82\x2C\xEB\x7E\xDF\xFC\x9D\xA2\xE0\x01\xF5\xF3\xEB\xB0\x1F\x31\xFB\xB1\xE2\x7C\x6F\x8B\x52\x7D\xE0\x74\x07\x0E\x7E\x7B\xEF\x73\x70\xBD\x50\x57\xF9\x72\x51\x89\x52\x03\x18\x58\x10\x11\x8A\x55\x0B\x5D\x51\xC8\x5A\x5C\x35\x54\x52\x64\x27\x52\xD4\x37\x06\xB2\x50\x57\x30\x52\x53\x82\x42\x05\x19\x87\x53\x5A\x17\x52\x3A\x52\x07\x1F\x8B\x51\x72\x51\x54\xFD\x50\x57\x20\x09\x54\x7A\x5D\x54\x7C\x58\x56\x54\x56\x55\x6D\x5B\x82\x2F\x52\x57\x5D\x5F\x55\xAE\x60\x83\x32\x8B\x54\x34\x80\x61\x68\x5A\x56\x5E\x5C\x56\x7D\x0A\x83\x5B\x52\x57\x74\x5C\x06\x2F\x84\x4E\x31\x84\x56\x43\x83\x5D\x7E\x57\x42\x4E\x89\x83\x9B\x7F\x3B\x78\x7F\x3B\x99\x6B\x77\x7A\x43\x10\x46\x7F\x3B\x80\x78\x81\x83\x7E\x5C\x4F\x74\x5F\x10\x6D\x51\xD5\x57\x12\xD7\x55\x3B\x40\x06\x7B\x33\x27\x80\xCF\x63\x6D\x35\x02\x71\x03\x14\x71\xD9\x62\x02\xDB\x65\x66\xE1\x6E\x66\xE3\x6F\x6D\xDD\x63\x6E\x3A\x75\x6E\x8A\x7D\x20\xE8\x61\x3A\xEA\x6D\x1E\x97\x61\x7D\x35\x0C\x72\x03\x1E\x72\x9F\x62\x02\xA1\x67\x78\x8B\x67\x68\x1D\x78\x86\x8A\x65\x73\x8C\x6B\x65\x8F\x6C\x73\x77\x67\x88\x3F\x4C\x33\x13\x5A\x44\x6A\x3E\x04\x6C\x32\x3F\x1F\x3C\x3E\x59\x83\x38\x01\x8D\x86\xBF\x39\x35\xC7\x4A\x3E\xE7\x30\x87\xB0\x32\x3A\xCA\x39\x35\x18\x09\x13\xAA\x87\x72\x77\x63\x41\xDC\x59\x35\xBF\x13\x3F\xEB\x37\x7E\x4F\x54\x8A\x48\x37\x8A\x0C\x1C\x01\xBA\x8B\x3E\xFF\x79\x4E\xC6\x82\x10\xA6\x83\x13\x59\x36\x28\xC2\x87\x3E\x1C\x06\x43\x4D\x39\x35\x44\x28\x8A\xDC\x82\x3F\x4D\x3A\x44\x46\x4D\x8D\x46\x4A\x44\x59\x33\x41\x9E\x3F\x8C\xF9\x35\x55\x65\x0B\x80\xE4\x4D\x8D\xFB\x60\x57\x59\x3C\x8E\xF7\x4D\x34\x43\x31\x8E\xDC\x83\x8E\xDC\x86\x8E\x9C\x13\x41\x46\x81\x12\xF2\x83\x25\xEE\x8B\x2F\xF0\x8A\x45\x3C\x13\x51\x4D\x35\x46\xF6\x88\x8A\xF8\x88\x8A\xFA\x85\x8C\x1B\x15\x8A\x56\x8F\x8F\xC6\x11\x90\x4E\x47\x14\xED\x84\x90\xFB\x43\x90\xF5\x23\x2E\xB8\x77\x44\xDD\x8B\x11\x46\x4B\x8D\x2D\x47\x3D\x02\x1A\x81\x16\x11\x8F\x0D\x5A\x91\xF2\x3B\x62\x29\x88\x92\x95\x26\x90\xE3\x2C\x6E\x1E\x9F\x91\x08\x11\x92\xDC\x8B\x6F\x26\x94\x07\x2E\x92\x8A\x2A\x9E\x3E\x2B\x6D\x81\x3B\x95\x31\x13\x55\x10\xE3\x24\x24\xD0\x44\x93\xE2\x3D\x34\x0B\x92\x3F\x44\x2D\x01\x20\x9C\x3A\x03\x1E\x94\x48\x9B\x4D\x42\x90\x57\x47\x93\x95\x50\x92\x3B\x97\x5C\x19\x58\x99\x13\x45\x93\x92\x5D\x9F\x31\x57\x9F\x31\x46\x43\x34\xDA\x87\x20\x0C\x16\x44\x65\x44\x4A\x58\x94\x8C\x52\x12\x92\x21\x3C\x32\x9E\x3D\x36\x76\x7F\x3B\xDF\x14\x69\xA8\x84\x24\xAA\x75\x93\x59\x90\x00\x7B\x93\x95\x6E\x95\x06\x09\x96\x2E\x2C\x3E\x5E\x74\x9A\x5D\xDC\x4C\x5D\x77\x99\x6E\x4C\x98\x3C\x04\x1A\x96\x8E\x98\x94\x81\x93\x98\x8E\x34\x98\x02\x16\x98\x7D\x9E\x36\x2F\x7E\x29\xDE\x8F\x95\xDA\x44\x93\x70\x92\x3F\xAD\x80\x5B\xBB\x57\x98\x78\x9C\x62\xB5\x3B\x98\x09\x16\x44\xB4\x67\x96\x86\x5B\x11\x63\x9F\x91\xA3\x96\x75\xB5\x36\x9A\x8C\x9E\x77\x65\x4B\x7D\x1F\x9B\x8A\x64\x04\x99\x88\x87\x9A\xC8\x69\x35\xB4\x84\x07\xAA\x90\x64\xDC\x84\x24\x9D\x71\x9B\xDD\x83\x9B\x8B\x7F\x8A\x99\x9B\x4F\xBB\x55\x46\x9D\x7D\x8D\xBC\x9E\x9B\x5A\x98\x98\xC1\x93\x92\x3C\x67\x8B\x58\x9D\x34\x0D\x81\x96\x59\x3D\x34\x01\x80\x9E\xE4\x93\x95\x4D\x36\x24\x1D\x95\x9E", "\xE2\x90\x23\xE5\x91\x9E\x9E\x9F\x4F\x61\x91\x9A\xEE\x3E\x8B\x77\x66\x9B\xBF\x9B\x99\x1D\x36\x44\xC5\x90\x00\xF3\x71\x3E\x44\x27\x48\xD1\x41\x96\xF4\x9D\x1E\xF6\x97\x9D\x89\x97\x67\x65\x40\xA0\x80\x9C\x8B\xED\x1A\x9C\x02\xA6\x76\xF9\x6B\x9F\xE4\x88\x4C\x1B\x12\x95\x01\xAD\x95\x03\xAC\x99\xCF\x99\x88\x9E\x25\x46\x16\xAB\xA0\x88\x98\x9F\x0E\xAD\x95\x81\x9D\x8D\xDF\x8F\x94\x5D\x9A\x44\x43\x35\x63\xD4\x9C\xA0\x9E\x23\x34\x1F\xA2\x99\xB9\x5C\x5A\x13\xA1\x97\x36\xA0\x00\xE7\x8F\x90\x03\x17\x83\xEB\x89\x92\x17\x98\x8A\xEF\x8F\xA3\x3C\x35\x95\xE4\x44\x8F\x28\xA8\x95\x2A\xA1\x99\x34\x9D\x7C\x90\x35\xA1\x61\x91\x98\x35\xA6\x99\x52\xAE\x39\xFC\x8B\x56\x12\x9D\x14\x14\x99\x30\x03\x1D\x93\x54\x9B\x4D\x30\x99\x13\x08\x91\x96\x49\xA6\x1E\x4F\xAE\xA2\x79\x00\xA3\x65\xA1\xA2\x34\xA9\x8F\x72\x9C\x19\xE8\x8F\x57\x6C\x07\xA5\x6C\x09\xA5\x4D\x40\x00\x5C\xA1\x8A\x44\xA3\x8F\xE3\x26\x3D\xBA\x92\x8E\x68\x45\x62\x69\xA7\x9A\xF8\x98\xA6\x24\xA3\xA3\x5C\x9D\x34\xB4\x6F\x97\x48\x95\x96\x4A\xAB\x9B\x66\xA4\xA8\x6D\x96\xA8\x4F\x59\x10\x1C\x97\xA4\x53\x93\xA6\x7D\x31\xA8\xB7\x9C\x5D\x90\xA0\xA2\x82\xAC\x5A\xF2\x98\x39\xFB\x29\x93\x41\x9E\xA5\x77\xA1\x3E\x2C\x9F\x28\xA5\xA1\x8A\x5F\xA7\x18\x32\x96\x9D\xE1\x38\xA9\xC7\x35\xA8\x21\xAC\xA9\x4E\xA4\xAB\x9F\xAC\x19\xA1\xAE\x42\x15\x90\x00\xA4\xA8\x91\x83\x57\xAA\x05\xA0\x00\x88\x39\x52\x67\x0B\xAA\x5B\x41\xAC\x42\xA9\x94\x31\x9C\x24\x61\x9C\xA8\x50\x31\x96\xC3\x9B\xAB\x00\x08\x8E\x4C\x73\xAC\x9A\x95\x9A\x4D\x34\x24\x59\x4D\x95\xD4\xA8\xAD\xDA\x23\x23\xC2\x9B\x2F\x15\x7B\x9F\xFB\x9D\x9F\xAE\x98\x10\x0A\xA3\x10\x23\xAB\x92\xB6\x80\x5B\xE3\xAD\xAB\x15\x79\xA0\xD2\xA4\xAE\xCA\x3D\xAE\x53\x9D\x8C\xC4\x9B\x94\xEE\x3D\x94\xB0\x9A\xA9\xBF\x9C\x9D\xBD\xA5\xA7\xF3\xA7\xAB\x58\x93\x9C\xF8\xA2\xA3\x6A\xAC\x95\xED\x9B\xAD\x1B\x17\x46\xED\x90\x9E\xE7\x9B\x11\xE9\x91\xB1\x4D\x32\x93\xE0\x44\x93\xD0\xA8\x10\x7D\xA7\x93\xF6\xA0\x5B\xD4\xA6\xAD\xC2\x20\xAE\x83\x55\x9A\x94\xA9\x13\xE0\x82\xA6\x7F\xAE\x97\x00\xB6\xA0\x83\xA6\xB0\x9E\xA7\x9B\x6B\xAC\x90\x6D\xAE\x90\x69\x3B\xA3\xEA\x82\xA7\x74\xA2\xAA\x16\x97\x4F\xC8\xA5\x90\x70\x56\xA4\x69\x9A\xB2\x37\xA4\x10\x8A\xAB\xA4\x8F\xA0\xB3\x0A\xB9\xAB\x36\xB2\x3F\x0D\x92\x15\x55\xA7\x84\x3A\xBE\xB1\xBD\xA7\xAA\x72\xA7\x90\x96\xA8\x94\xB2\xA3\x10\x47\xBE\xA8\xB5\xAD\x8B\x4A\xB8\xB4\x0B\xBD\xB4\xF3\x95\xB3\x52\x1F\xA6\x11\x9E\xA3\x3E\xB4\xB5\x5A\xAD\xB3\x00\x90\xAC\x88\x55\xA4\x7B\xAB\x11\x1D\xBA\x90\x2B\xBD\xB5\x2D\xAF\xB5\x4D\xAC\x10\x31\xA2\xB6\x4C\xB6\x40\x88\xAB\x11\x78\xB7\x8F\x77\xBD\xB2\xC4\xA0\x7D\x2F\xA1\xB6\x5E\xBF\xB7\xC3\x23\x2E\xDF\x91\xB1\x1B\x99\x13\x1D\x90\xAB\x59\x3B\xB5\x02\x13\xB8\xA8\x8C\xA4\x89\xBC\xB7\x86\xBA\x99\xA0\xAC\xB6\x75\xAF\xAB\x9F\x99\xAC\x71\xBB\xB6\x99\xB2\x3A\x2D\x90\xB7\xAD\xA9\x22\xAF\xA4\xB4\x52\xA6\xB4\x9D\xB8\xB8\x67\xAA\xB8\x79\xBC\xB8\xD4\xA8\x93\x26\x8F\xB3\x19\x9B\xAC\xE9\x3F\x93\xC6\xAA\xBB\xCA\xA6\xBA\xE1\x3D\x34\x9D\x7C\x96\x90\xB7\xBC\x29\x23\x2E\x29\xB9\xA2\x2B\xB6\xBC\x4B\xB7\x9B\xBB\x5D\xA9\xCF\xB7\x9F\x33\xBE\xB4\x66\xB5\x06\xE8\x8C\xA3\x53\xB0\xA4\x02\x9C\xBB\x78\xA6\xAA\x72\xB2\x34\x59\xB4\x93\x96\xB0\x00\xCE\xBE\xB7\xD0\xB5\x3B\xD2\xB8\xBE\xD4\xBE\xA6\x6C\xA7\xA3\x54\xAC\x32\xFD\x8D\xA3\x2F\x92\xAC\xDA\x9B\xA5\xDE\xB4\xBA\x79\xA3\x25\x58\xB3\xB4\xCC\xB5\xB4\xA3\x41\xBB\x25\xBA\xBE\xB4\xBC\x8D\x50\xAF\xBE\x53\xAE\xBE\x37\xB0\x8D\x69\xB5\xBF\xF9\xB1\xAF\x6D\xB6\xA7\xF9\xB7\xB5\x73\xBF\x9A\x5D\x9B\xB1\x04\x17\xBE\x8B\xB9\xBE\xBF\x3B\xBE\x1B\xCD\xBE\x93\xA1\xB8\xAB\x77\xC1\xCD\xB2\xC0\xA8\x9D\xC1\x05\xC7\xBA\x63\xB6\x40\x1F\x35\xA9\xFE\xB8\xA4\x25\xC8\xAB\x1C\xCD\x3C\x29\xCE\xAE\x2B\xC5\x06\xB7\xB3\xAA\xB9\xBA\xBA\xF6\xB8\x98\x8C\x99\xBA\xA3\xB0\xB4\xE1\xBE\xAA\xE3\xBF\x91\xE5\xBA\xC1\xB5\xB3\xC3\x84\x35\xC3\x3E\x92\xA9\x6F\x90\xBA\x3C\xBE\xAB\x3B\xC1\xC4\xBB\xB2\xBC\x4D\xC8\x98\x40\x9C\xC3\x43\xC9\x22\x59\x31\x10\xC8\xBD\xB0\xCD\xAA\x24\xF1\x94\xA1\xA8\x82\xB2\x14\x14\xB2\x27\xC3\x38\xD9\x86\x1F\x5F\xC4\x93\x4D\x3C\xA7\x6D\xC9\xBC\x39\x19\x35\xAD\x41\xC7\x6E\xC3\x2E\x89\xA6\xC7\x44\x93\xC7\x4C\xCA\xC7\x7B\xC7\x18\xCB\xB8\x95\xB1\x9E\xC1\x49\xC0\xC2\x67\xB8\xC0\x95\x99\xA3\x0B\xC9\x8E\x52\x1B\xBD\xF2\x31\xA4\x56\xCD\xA5\xAC\xA1\xB4\x2B", "\x3F\xC2\x53\x93\xC8\x7D\xC2\x3E\x07\xC4\xB3\xF0\xBB\x8F\xF2\xB6\xA5\x6A\xBF\xB6\xDC\xB5\xB5\x13\xC1\xCA\xC6\x1D\xBF\x7C\x9D\x95\x98\xCC\xB9\x32\xC6\xC8\xD8\xB8\xC8\x69\x4F\x3C\x0A\xC8\x10\x10\x98\x42\x8E\xCE\x3E\x90\xC2\xCA\x43\xA0\xBE\x7A\xA9\x13\x70\xCE\xAD\x2B\xBD\xB7\x1F\xCE\xB2\xAC\x5F\xC7\x29\x29\xC7\xC0\xC0\xC0\x02\x12\xCC\x85\xC4\xCC\x0C\xB6\xCC\xF5\x23\xB9\x61\x9B\xCA\x08\x1C\xCC\x06\xCE\xC4\x38\xC0\xC5\xBC\xA2\xC5\xDA\x30\xBC\xA5\xBA\xCB\x57\xC7\x9A\x40\xCB\xCB\x93\xCB\xC5\xF5\x2D\xBA\xAA\xC1\xCC\x26\xC3\x68\x39\xCD\xAB\xA2\xB4\xCE\xA4\xB3\x94\xD7\xC8\xC5\xBF\xBA\xC5\xBD\xC7\x18\x59\x3B\xAE\x7E\xCC\xAC\xBE\xC4\xB7\x63\xCA\xCC\x00\x0A\xCF\x1F\x9A\xAF\xD0\xC9\x10\xC8\xC8\x95\xC9\x46\x48\x0F\xA7\xA3\x6C\x7A\x93\xFB\xC5\x9E", }; void JsonGetParserBuffer(vl::stream::MemoryStream& stream) { vl::stream::MemoryStream compressedStream; for (vint i = 0; i < parserBufferRows; i++) { vint size = i == parserBufferRows - 1 ? parserBufferRemain : parserBufferBlock; compressedStream.Write((void*)parserBuffer[i], size); } compressedStream.SeekFromBegin(0); vl::stream::LzwDecoder decoder; vl::stream::DecoderStream decoderStream(compressedStream, decoder); vl::collections::Array buffer(65536); while (true) { vl::vint size = decoderStream.Read(&buffer[0], 65536); if (size == 0) break; stream.Write(&buffer[0], size); } stream.SeekFromBegin(0); } /*********************************************************************** Unescaping Function Foward Declarations ***********************************************************************/ extern void JsonUnescapingString(vl::parsing::ParsingToken& value, const vl::collections::List& tokens); /*********************************************************************** Parsing Tree Conversion Driver Implementation ***********************************************************************/ class JsonTreeConverter : public vl::parsing::ParsingTreeConverter { public: using vl::parsing::ParsingTreeConverter::SetMember; bool SetMember(JsonLiteral::JsonValue& member, vl::Ptr node, const TokenList& tokens) { vl::Ptr token=node.Cast(); if(token) { if(token->GetValue()==L"True") { member=JsonLiteral::JsonValue::True; return true; } else if(token->GetValue()==L"False") { member=JsonLiteral::JsonValue::False; return true; } else if(token->GetValue()==L"Null") { member=JsonLiteral::JsonValue::Null; return true; } else { member=JsonLiteral::JsonValue::True; return false; } } member=JsonLiteral::JsonValue::True; return false; } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->value, obj->GetMember(L"value"), tokens); } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { if(SetMember(tree->content, obj->GetMember(L"content"), tokens)) { JsonUnescapingString(tree->content, tokens); } } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->content, obj->GetMember(L"content"), tokens); } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->items, obj->GetMember(L"items"), tokens); } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { if(SetMember(tree->name, obj->GetMember(L"name"), tokens)) { JsonUnescapingString(tree->name, tokens); } SetMember(tree->value, obj->GetMember(L"value"), tokens); } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->fields, obj->GetMember(L"fields"), tokens); } vl::Ptr ConvertClass(vl::Ptr obj, const TokenList& tokens)override { if(obj->GetType() == L"Literal") { auto tree = vl::Ptr(new JsonLiteral); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"String") { auto tree = vl::Ptr(new JsonString); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Number") { auto tree = vl::Ptr(new JsonNumber); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Array") { auto tree = vl::Ptr(new JsonArray); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"ObjectField") { auto tree = vl::Ptr(new JsonObjectField); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Object") { auto tree = vl::Ptr(new JsonObject); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else return 0; } }; vl::Ptr JsonConvertParsingTreeNode(vl::Ptr node, const vl::collections::List& tokens) { JsonTreeConverter converter; vl::Ptr tree; converter.SetMember(tree, node, tokens); return tree; } /*********************************************************************** Parsing Tree Conversion Implementation ***********************************************************************/ vl::Ptr JsonLiteral::Convert(vl::Ptr node, const vl::collections::List& tokens) { return JsonConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr JsonString::Convert(vl::Ptr node, const vl::collections::List& tokens) { return JsonConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr JsonNumber::Convert(vl::Ptr node, const vl::collections::List& tokens) { return JsonConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr JsonArray::Convert(vl::Ptr node, const vl::collections::List& tokens) { return JsonConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr JsonObjectField::Convert(vl::Ptr node, const vl::collections::List& tokens) { return JsonConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr JsonObject::Convert(vl::Ptr node, const vl::collections::List& tokens) { return JsonConvertParsingTreeNode(node, tokens).Cast(); } /*********************************************************************** Parser Function ***********************************************************************/ vl::Ptr JsonParseAsParsingTreeNode(const vl::WString& input, vl::Ptr table, vl::collections::List>& errors, vl::vint codeIndex) { vl::parsing::tabling::ParsingState state(input, table, codeIndex); state.Reset(L"JRoot"); vl::Ptr parser=vl::parsing::tabling::CreateStrictParser(table); vl::Ptr node=parser->Parse(state, errors); return node; } vl::Ptr JsonParseAsParsingTreeNode(const vl::WString& input, vl::Ptr table, vl::vint codeIndex) { vl::collections::List> errors; return JsonParseAsParsingTreeNode(input, table, errors, codeIndex); } vl::Ptr JsonParse(const vl::WString& input, vl::Ptr table, vl::collections::List>& errors, vl::vint codeIndex) { vl::parsing::tabling::ParsingState state(input, table, codeIndex); state.Reset(L"JRoot"); vl::Ptr parser=vl::parsing::tabling::CreateStrictParser(table); vl::Ptr node=parser->Parse(state, errors); if(node && errors.Count()==0) { return JsonConvertParsingTreeNode(node, state.GetTokens()).Cast(); } return 0; } vl::Ptr JsonParse(const vl::WString& input, vl::Ptr table, vl::vint codeIndex) { vl::collections::List> errors; return JsonParse(input, table, errors, codeIndex); } /*********************************************************************** Table Generation ***********************************************************************/ vl::Ptr JsonLoadTable() { vl::stream::MemoryStream stream; JsonGetParserBuffer(stream); auto table = vl::Ptr(new vl::parsing::tabling::ParsingTable(stream)); table->Initialize(); return table; } } } } /*********************************************************************** .\XML\PARSINGXML.CPP ***********************************************************************/ namespace vl { namespace parsing { namespace xml { using namespace stream; using namespace collections; using namespace regex; /*********************************************************************** Unescaping Function Foward Declarations ***********************************************************************/ void XmlMergeTextFragment(vl::collections::List>& value, const vl::collections::List& tokens) { vint begin=-1; vint end=-1; for(vint i=value.Count()-1;i>=-1;i--) { if(i==-1) { if(end!=-1) begin=0; } else if(value[i].Cast()) { if(end==-1) end=i; } else { if(end!=-1) begin=i+1; } if(begin!=-1 && end!=-1) { vint tokenBegin=value[begin].Cast()->content.tokenIndex; vint tokenEnd=value[end].Cast()->content.tokenIndex; while(tokenBegin>0) { if(tokens.Get(tokenBegin-1).token==(vint)XmlParserTokenIndex::SPACE || tokens.Get(tokenBegin-1).token==-1) { tokenBegin--; } else { break; } } while(tokenEndcodeRange=range; xmlText->content.codeRange=range; xmlText->content.value=XmlUnescapeValue(text); value.RemoveRange(begin, end-begin+1); value.Insert(begin, xmlText); begin=-1; end=-1; } } } void XmlUnescapeAttributeValue(vl::parsing::ParsingToken& value, const vl::collections::List& tokens) { value.value=XmlUnescapeValue(value.value.Sub(1, value.value.Length()-2)); } void XmlUnescapeCData(vl::parsing::ParsingToken& value, const vl::collections::List& tokens) { value.value=XmlUnescapeCData(value.value); } void XmlUnescapeComment(vl::parsing::ParsingToken& value, const vl::collections::List& tokens) { value.value=XmlUnescapeComment(value.value); } /*********************************************************************** XmlPrintVisitor ***********************************************************************/ class XmlPrintVisitor : public Object, public XmlNode::IVisitor { public: TextWriter& writer; XmlPrintVisitor(TextWriter& _writer) :writer(_writer) { } void Visit(XmlText* node) { writer.WriteString(XmlEscapeValue(node->content.value)); } void Visit(XmlCData* node) { writer.WriteString(XmlEscapeCData(node->content.value)); } void Visit(XmlAttribute* node) { writer.WriteString(node->name.value); writer.WriteString(L"=\""); writer.WriteString(XmlEscapeValue(node->value.value)); writer.WriteString(L"\""); } void Visit(XmlComment* node) { writer.WriteString(XmlEscapeComment(node->content.value)); } void Visit(XmlElement* node) { writer.WriteChar(L'<'); writer.WriteString(node->name.value); for (auto att : node->attributes) { writer.WriteChar(L' '); att->Accept(this); } if(node->subNodes.Count()==0) { writer.WriteString(L"/>"); } else { writer.WriteChar(L'>'); for (auto subNode : node->subNodes) { subNode->Accept(this); } writer.WriteString(L"name.value); writer.WriteChar(L'>'); } } void Visit(XmlInstruction* node) { writer.WriteString(L"name.value); for (auto att : node->attributes) { writer.WriteChar(L' '); att->Accept(this); } writer.WriteString(L"?>"); } void Visit(XmlDocument* node) { for (auto prolog : node->prologs) { prolog->Accept(this); } node->rootElement->Accept(this); } }; /*********************************************************************** API ***********************************************************************/ WString XmlEscapeValue(const WString& value) { WString result; const wchar_t* reading=value.Buffer(); while(wchar_t c=*reading++) { switch(c) { case L'<': result+=L"<"; break; case L'>': result+=L">"; break; case L'&': result+=L"&"; break; case L'\'': result+=L"'"; break; case L'\"': result+=L"""; break; default: result+=WString::FromChar(c); } } return result; } WString XmlUnescapeValue(const WString& value) { WString result; const wchar_t* reading=value.Buffer(); while(*reading) { if(wcsncmp(reading, L"<", 4)==0) { result+=WString::FromChar(L'<'); reading+=4; } else if(wcsncmp(reading, L">", 4)==0) { result+=WString::FromChar(L'>'); reading+=4; } else if(wcsncmp(reading, L"&", 5)==0) { result+=WString::FromChar(L'&'); reading+=5; } else if(wcsncmp(reading, L"'", 6)==0) { result+=WString::FromChar(L'\''); reading+=6; } else if(wcsncmp(reading, L""", 6)==0) { result+=WString::FromChar(L'\"'); reading+=6; } else { result+=WString::FromChar(*reading++); } } return result; } WString XmlEscapeCData(const WString& value) { return L""; } WString XmlUnescapeCData(const WString& value) { return value.Sub(9, value.Length()-12); } WString XmlEscapeComment(const WString& value) { return L""; } WString XmlUnescapeComment(const WString& value) { return value.Sub(4, value.Length()-7); } void XmlPrint(Ptr node, stream::TextWriter& writer) { XmlPrintVisitor visitor(writer); node->Accept(&visitor); } void XmlPrintContent(Ptr element, stream::TextWriter& writer) { XmlPrintVisitor visitor(writer); for (auto node : element->subNodes) { node->Accept(&visitor); } } WString XmlToString(Ptr node) { return GenerateToStream([&](StreamWriter& writer) { XmlPrint(node, writer); }); } /*********************************************************************** Linq To Xml ***********************************************************************/ Ptr XmlGetAttribute(Ptr element, const WString& name) { return XmlGetAttribute(element.Obj(), name); } Ptr XmlGetElement(Ptr element, const WString& name) { return XmlGetElement(element.Obj(), name); } collections::LazyList> XmlGetElements(Ptr element) { return XmlGetElements(element.Obj()); } collections::LazyList> XmlGetElements(Ptr element, const WString& name) { return XmlGetElements(element.Obj(), name); } WString XmlGetValue(Ptr element) { return XmlGetValue(element.Obj()); } Ptr XmlGetAttribute(XmlElement* element, const WString& name) { for (auto att : element->attributes) { if(att->name.value==name) { return att; } } return 0; } Ptr XmlGetElement(XmlElement* element, const WString& name) { for (auto node : element->subNodes) { Ptr subElement=node.Cast(); if(subElement && subElement->name.value==name) { return subElement; } } return 0; } collections::LazyList> XmlGetElements(XmlElement* element) { return From(element->subNodes) .FindType(); } collections::LazyList> XmlGetElements(XmlElement* element, const WString& name) { return From(element->subNodes) .FindType() .Where([name](auto&& e){return e->name.value==name;}); } WString XmlGetValue(XmlElement* element) { WString result; for (auto node : element->subNodes) { if(Ptr text=node.Cast()) { result+=text->content.value; } else if(Ptr text=node.Cast()) { result+=text->content.value; } } return result; } /*********************************************************************** XmlElementWriter ***********************************************************************/ XmlElementWriter::XmlElementWriter(Ptr _element, const XmlElementWriter* _previousWriter) :element(_element) ,previousWriter(_previousWriter) { } XmlElementWriter::~XmlElementWriter() { } const XmlElementWriter& XmlElementWriter::Attribute(const WString& name, const WString& value)const { auto node = Ptr(new XmlAttribute); node->name.value=name; node->value.value=value; element->attributes.Add(node); return *this; } XmlElementWriter XmlElementWriter::Element(const WString& name)const { auto node = Ptr(new XmlElement); node->name.value=name; element->subNodes.Add(node); return XmlElementWriter(node, this); } const XmlElementWriter& XmlElementWriter::End()const { return *previousWriter; } const XmlElementWriter& XmlElementWriter::Text(const WString& value)const { auto node = Ptr(new XmlText); node->content.value=value; element->subNodes.Add(node); return *this; } const XmlElementWriter& XmlElementWriter::CData(const WString& value)const { auto node = Ptr(new XmlCData); node->content.value=value; element->subNodes.Add(node); return *this; } const XmlElementWriter& XmlElementWriter::Comment(const WString& value)const { auto node = Ptr(new XmlComment); node->content.value=value; element->subNodes.Add(node); return *this; } } } } /*********************************************************************** .\XML\PARSINGXML_AST.CPP ***********************************************************************/ /*********************************************************************** This file is generated by: Vczh Parser Generator From parser definition:ParsingXml.parser.txt Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { namespace xml { /*********************************************************************** Visitor Pattern Implementation ***********************************************************************/ void XmlText::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } void XmlCData::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } void XmlAttribute::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } void XmlComment::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } void XmlElement::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } void XmlInstruction::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } void XmlDocument::Accept(XmlNode::IVisitor* visitor) { visitor->Visit(this); } } } } namespace vl { namespace reflection { namespace description { #ifndef VCZH_DEBUG_NO_REFLECTION using namespace vl::parsing::xml; #define PARSING_TOKEN_FIELD(NAME)\ CLASS_MEMBER_EXTERNALMETHOD_TEMPLATE(get_##NAME, NO_PARAMETER, vl::WString(ClassType::*)(), [](ClassType* node) { return node->NAME.value; }, L"*", L"*")\ CLASS_MEMBER_EXTERNALMETHOD_TEMPLATE(set_##NAME, { L"value" }, void(ClassType::*)(const vl::WString&), [](ClassType* node, const vl::WString& value) { node->NAME.value = value; }, L"*", L"*")\ CLASS_MEMBER_PROPERTY_REFERENCETEMPLATE(NAME, get_##NAME, set_##NAME, L"$This->$Name.value")\ IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlNode, system::XmlNode) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlText, system::XmlText) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlCData, system::XmlCData) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlAttribute, system::XmlAttribute) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlComment, system::XmlComment) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlElement, system::XmlElement) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlInstruction, system::XmlInstruction) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlDocument, system::XmlDocument) IMPL_TYPE_INFO_RENAME(vl::parsing::xml::XmlNode::IVisitor, system::XmlNode::IVisitor) #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA BEGIN_CLASS_MEMBER(XmlNode) CLASS_MEMBER_METHOD_OVERLOAD(Accept, {L"visitor"}, void(XmlNode::*)(XmlNode::IVisitor* visitor)) END_CLASS_MEMBER(XmlNode) BEGIN_CLASS_MEMBER(XmlText) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(content) END_CLASS_MEMBER(XmlText) BEGIN_CLASS_MEMBER(XmlCData) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(content) END_CLASS_MEMBER(XmlCData) BEGIN_CLASS_MEMBER(XmlAttribute) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(name) PARSING_TOKEN_FIELD(value) END_CLASS_MEMBER(XmlAttribute) BEGIN_CLASS_MEMBER(XmlComment) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(content) END_CLASS_MEMBER(XmlComment) BEGIN_CLASS_MEMBER(XmlElement) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(name) PARSING_TOKEN_FIELD(closingName) CLASS_MEMBER_FIELD(attributes) CLASS_MEMBER_FIELD(subNodes) END_CLASS_MEMBER(XmlElement) BEGIN_CLASS_MEMBER(XmlInstruction) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) PARSING_TOKEN_FIELD(name) CLASS_MEMBER_FIELD(attributes) END_CLASS_MEMBER(XmlInstruction) BEGIN_CLASS_MEMBER(XmlDocument) CLASS_MEMBER_BASE(XmlNode) CLASS_MEMBER_CONSTRUCTOR(vl::Ptr(), NO_PARAMETER) CLASS_MEMBER_FIELD(prologs) CLASS_MEMBER_FIELD(rootElement) END_CLASS_MEMBER(XmlDocument) BEGIN_INTERFACE_MEMBER(XmlNode::IVisitor) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlText* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlCData* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlAttribute* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlComment* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlElement* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlInstruction* node)) CLASS_MEMBER_METHOD_OVERLOAD(Visit, {L"node"}, void(XmlNode::IVisitor::*)(XmlDocument* node)) END_INTERFACE_MEMBER(XmlNode) #endif #undef PARSING_TOKEN_FIELD #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA class XmlTypeLoader : public vl::Object, public ITypeLoader { public: void Load(ITypeManager* manager) { ADD_TYPE_INFO(vl::parsing::xml::XmlNode) ADD_TYPE_INFO(vl::parsing::xml::XmlText) ADD_TYPE_INFO(vl::parsing::xml::XmlCData) ADD_TYPE_INFO(vl::parsing::xml::XmlAttribute) ADD_TYPE_INFO(vl::parsing::xml::XmlComment) ADD_TYPE_INFO(vl::parsing::xml::XmlElement) ADD_TYPE_INFO(vl::parsing::xml::XmlInstruction) ADD_TYPE_INFO(vl::parsing::xml::XmlDocument) ADD_TYPE_INFO(vl::parsing::xml::XmlNode::IVisitor) } void Unload(ITypeManager* manager) { } }; #endif #endif bool XmlLoadTypes() { #ifdef VCZH_DESCRIPTABLEOBJECT_WITH_METADATA ITypeManager* manager = GetGlobalTypeManager(); if(manager) { auto loader = Ptr(new XmlTypeLoader); return manager->AddTypeLoader(loader); } #endif return false; } } } } /*********************************************************************** .\XML\PARSINGXML_PARSER.CPP ***********************************************************************/ /*********************************************************************** This file is generated by: Vczh Parser Generator From parser definition:ParsingXml.parser.txt Licensed under https://github.com/vczh-libraries/License ***********************************************************************/ namespace vl { namespace parsing { namespace xml { /*********************************************************************** ParserText ***********************************************************************/ const wchar_t* const parserTextBuffer[] = { L"" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"// AST" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"" L"\r\n" , L"class Node\t\t\t\t\t\t\t\t\t\t@Document(\"Base class of XML nodes.\")" L"\r\n" , L"{" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Text : Node\t\t\t\t\t\t\t\t@Document(\"Text node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken content\t\t\t\t\t\t\t\t@Document(\"Content of the text node.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class CData : Node\t\t\t\t\t\t\t\t@Document(\"CData node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken content (XmlUnescapeCData)\t\t\t@Document(\"Content of the cdata node\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Attribute : Node\t\t\t\t\t\t\t@Document(\"Attribute node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken name\t\t\t\t\t\t\t\t\t@Document(\"Attribute name.\"), @Color(\"AttName\");" L"\r\n" , L"\ttoken value (XmlUnescapeAttributeValue)\t\t@Document(\"Attribute value.\"), @Color(\"AttValue\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Comment : Node\t\t\t\t\t\t\t@Document(\"Comment node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken content (XmlUnescapeComment)\t\t\t@Document(\"Content of the comment node.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Element : Node\t\t\t\t\t\t\t@Document(\"Element node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken name\t\t\t\t\t\t\t\t\t@Document(\"Element name of the open element node.\"), @Color(\"TagName\");" L"\r\n" , L"\ttoken closingName\t\t\t\t\t\t\t@Document(\"(Optional): Element name of the closing element node. The name is ignored when serializing XML to text.\"), @Color(\"TagName\");" L"\r\n" , L"\tAttribute[] attributes\t\t\t\t\t\t@Document(\"Attributes of the element.\");" L"\r\n" , L"\tNode[] subNodes (XmlMergeTextFragment)\t\t@Document(\"Sub nodes for element nodes, text nodes, cdata nodes and comment nodes.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Instruction : Node\t\t\t\t\t\t@Document(\"Instruction node.\")" L"\r\n" , L"{" L"\r\n" , L"\ttoken name\t\t\t\t\t\t\t\t\t@Document(\"Name of the instruction.\"), @Color(\"TagName\");" L"\r\n" , L"\tAttribute[] attributes\t\t\t\t\t\t@Document(\"Attributes of the instruction.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"class Document : Node\t\t\t\t\t\t\t@Document(\"XML document node.\")" L"\r\n" , L"{" L"\r\n" , L"\tNode[] prologs\t\t\t\t\t\t\t\t@Document(\"Prologue nodes, for instructions and comments.\");" L"\r\n" , L"\tElement rootElement\t\t\t\t\t\t\t@Document(\"Root element of the XML document.\");" L"\r\n" , L"}" L"\r\n" , L"" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"// Lexer" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"" L"\r\n" , L"token INSTRUCTION_OPEN = \"/\"\t\t@Color(\"Boundary\");" L"\r\n" , L"token COMPLEX_ELEMENT_OPEN = \"/\"\t\t@Color(\"Boundary\");" L"\r\n" , L"token ELEMENT_OPEN = \"/<\"\t\t\t\t@Color(\"Boundary\");" L"\r\n" , L"token ELEMENT_CLOSE = \"/>\"\t\t\t\t@Color(\"Boundary\");" L"\r\n" , L"token EQUAL = \"/=\";" L"\r\n" , L"" L"\r\n" , L"token NAME = \"[a-zA-Z0-9:._/-]+\"\t\t\t\t\t\t\t\t@ContextColor();" L"\r\n" , L"token ATTVALUE = \"\"\"[^<>\"\"]*\"\"|\'[^<>\']*\'\"\t\t\t\t\t\t@ContextColor();" L"\r\n" , L"token COMMENT = \"/]|-[^/->]|--[^>])*--/>\"\t\t\t@Color(\"Comment\");" L"\r\n" , L"token CDATA = \"/])*/]/]/>\";" L"\r\n" , L"token TEXT = \"([^<>=\"\"\' /r/n/ta-zA-Z0-9:._/-])+|\"\"|\'\";" L"\r\n" , L"" L"\r\n" , L"discardtoken SPACE = \"/s+\";" L"\r\n" , L"" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"// Rules" L"\r\n" , L"//////////////////////////////////////////////////////////////////" L"\r\n" , L"" L"\r\n" , L"rule Attribute XAttribute = NAME:name \"=\" ATTVALUE:value as Attribute;" L"\r\n" , L"rule Text XText = (NAME:content | EQUAL:content | ATTVALUE:content | TEXT:content) as Text;" L"\r\n" , L"rule CData XCData = CDATA:content as CData;" L"\r\n" , L"rule Comment XComment = COMMENT:content as Comment;" L"\r\n" , L"rule Element XElement = \"<\" NAME:name {XAttribute:attributes} (\"/>\" | \">\" {XSubNode:subNodes} \"\") as Element;" L"\r\n" , L"rule Node XSubNode = !XText | !XCData | !XComment | !XElement;" L"\r\n" , L"rule Instruction XInstruction = \"\" as Instruction;" L"\r\n" , L"rule Document XDocument = {XInstruction:prologs | XComment:prologs} XElement:rootElement as Document;" L"\r\n" }; const vint lengthTextBuffer[] = { 2, 68, 8, 68, 2, 59, 3, 3, 2, 50, 3, 63, 3, 2, 52, 3, 77, 3, 2, 59, 3, 70, 94, 3, 2, 55, 3, 82, 3, 2, 55, 3 , 93, 163, 71, 128, 3, 2, 62, 3, 79, 75, 3, 2, 61, 3, 85, 76, 3, 2, 68, 10, 68, 2, 55, 55, 58, 58, 50, 51, 21, 2, 58, 65 , 75, 63, 56, 2, 29, 2, 68, 10, 68, 2, 72, 93, 45, 53, 134, 64, 93, 103 }; const vint lengthTextBufferTotal = 3474; vl::WString XmlGetParserTextBuffer() { vl::collections::Array textBuffer(lengthTextBufferTotal + 1); wchar_t* reading = &textBuffer[0]; for(vint i = 0; i < sizeof(parserTextBuffer) / sizeof(*parserTextBuffer); i++) { memcpy(reading, parserTextBuffer[i], lengthTextBuffer[i] * sizeof(wchar_t)); reading += lengthTextBuffer[i]; } *reading = 0; return &textBuffer[0]; } /*********************************************************************** SerializedTable ***********************************************************************/ const vint parserBufferLength = 5096; // 20987 bytes before compressing const vint parserBufferBlock = 1024; const vint parserBufferRemain = 1000; const vint parserBufferRows = 5; const char* const parserBuffer[] = { "\x00\x1E\x00\x02\x83\x80\x07\x7D\x00\x82\x03\xFF\x48\x08\x82\x83\x86\x80\x22\x6F\x63\x35\x2D\x35\x36\x34\x3B\x83\x0C\x98\x0C\x80\x0A\x8A\x82\x21\x30\x73\x65\x20\x03\x3C\x30\x93\x38\x10\x6F\x66\x20\x18\x2D\x24\x24\x10\x37\x6F\x64\x25\x33\x3E\x17\x7E\x01\x8D\x0A\x9B\x8E\x80\x00\x8A\x88\x8A\x8B\x18\x88\x8A\x1B\x83\xA2\x80\x2B\x32\x78\x74\x2F\x91\x95\x35\x9B\x9C\x82\x3B\xBD\x93\x95\x87\x8B\x8D\x8E\x84\x57\x9F\xA1\x83\x95\x93\x3B\x93\x94\x2B\xAD\x8B\xB2\x94\x9E\x98\x9D\xA7\x03\xD1\x91\x93\xA0\xA3\x8F\x04\xB7\x03\xC3\x04\x21\x34\x39\x33\xB1\xA6\x39\x8C\xAA\xBE\x94\xA9\xA0\x83\xAB\x7F\x83\x9A\xA2\x94\x96\x90\x94\x95\x2C\xAE\xB0\x84\xB5\x9F\x98\x87\xBC\x08\xFB\xAC\xB5\xAC\x87\x07\x8C\xCB\x02\xC1\x34\x34\x32\x39\x36\x31\x3A\x74\x22\x8B\xD8\xB8\xB5\x87\x80\xA9\x3F\x94\xD8\xB6\xA2\xC0\x90\xC3\xAE\x5F\x88\xE2\xA3\xD5\xB6\xC4\x84\xC8\x50\xA7\xEB\xA9\xDE\xBB\x81\x07\x8D\xBF\x82\x83\x2F\x3D\x31\xD7\xBA\xA7\x3A\xBA\xFC\xAD\xB9\x8C\xD6\x8E\xD7\x5C\x86\xE0\xA9\xC7\xBD\xDB\xB3\xE4\x0F\xBB\xFD\xA0\xEC\xEA\x81\x22\x36\x65\x46\xE3\xC8\xEA\xBA\xE7\xC9\xDE\x02\x80\xC3\x99\xA0\xED\xC2\xAF\xC3\x61\x8A\xCC\xB6\xEF\xC5\xD0\x00\xC9\xBC\x9B\x91\x1A\xF3\x81\x26\x37\x39\x9C\x75\x13\x18\x1D\x69\x2F\x1E\x1B\x71\xB8\x69\x56\x79\x7E\xCD\x4D\x80\x03\x60\xD1\x6F\x73\x77\x6C\xF3\x4D\x67\x75\x79\xD9\x4B\x7A\x68\x41\x12\x1B\x4D\x86\x40\xD4\x64\x1A\x76\x45\x08\xB6\x78\x7F\x76\x0E\x81\x6F\x73\x56\xEE\x71\x61\x7D\x75\x35\x68\x75\x42\x89\x0B\xA8\x88\x41\x06\x1B\x78\x82\x72\x81\xA1\x64\x80\x58\x1D\x68\x22\x41\x69\x52\x08\xB2\x89\x82\x69\x19\xA7\x79\x8F\x92\x3B\x96\x44\x54\x7C\x40\x84\x64\x18\x5D\x76\x63\x66\x59\x7D\xD8\x7C\x43\x8B\x86\x03\x58\x68\x42\x97\x9A\x5C\x6E\x64\x68\xA2\x61\x14\x45\x4D\x05\x1B\x49\x9B\x8E\x6C\x2F\x12\x1E\x8C\x07\x1B\x41\x9D\x66\x9B\x4E\x16\x99\x19\x57\xB7\x64\x8D\x92\x0C\x88\x40\x07\x46\x7F\xB4\x92\x9B\x67\xA1\x60\x06\x1D\x18\x6C\x35\x18\x5F\x9A\x08\x4C\xA3\x71\x9B\x6F\x99\x99\x6E\xA0\x56\x08\xAA\xA2\x41\x17\x91\x65\x8C\x73\x08\x5C\x0B\x46\xA7\x4D\xBD\x8A\x53\x8F\x41\xA3\x44\x72\x78\x4C\x56\x88\x94\x86\x5A\x9A\x82\x8E\x09\x1B\x70\xAE\x74\x78\xE2\x77\x90\x95\xA9\x6F\x30\x15\x44\x08\x65\x34\xAF\x93\x78\xFF\x0C\xA4\x42\xA3\x6F\x10\xA0\x9E\x9C\x03\x54\x11\x1B\x19\x76\x94\x49\x9C\x41\x47\xB7\x7B\xA4\x97\x02\x67\x1B\x46\xB5\x02\x68\x0F\x10\x1C\x04\x86\x88\xA1\x0A\x3A\x20\x0F\x75\x78\xBF\xB6\xA4\xAB\x6B\x6F\x33\x19\x1A\x1B\x67\x3C\xAE\xAC\x89\xE3\x60\x04\x15\xA9\x6E\x36\xA9\x1B\x49\x69\x27\x10\x4E\x1C\x65\x24\x10\x0B\x1D\x41\x87\x81\x49\x67\x88\xA9\x1A\x1F\xBA\xED\x94\x74\x1F\x1B\x20\x03\x94\x1C\x9A\x6A\xB1\x55\xB2\x9B\xC7\x8C\x43\x9C\x00\xCA\x8C\xB7\x9C\xA6\xAB\x9A\xAC\x9E\x8D\x04\x5A\x0B\x46\xC8\x82\x9D\x64\xA3\x4C\xB7\xA2\x4D\xAC\xB9\x16\x71\x89\xA7\xA4\xAD\x8A\x9C\x43\x11\x1B\x74\xC2\x43\x14\x75\x22\x15\x74\x08\x66\x2E\x9E\xB8\xCB\x4A\x74\x6C\x09\xC3\x48\x41\xD3\x7F\xD0\x63\x13\x95\x5F\xCE\x61\x2E\x1E\xBF\x18\xA7\xBF\xA5\x87\xB3\x09\x7B\x91\xCD\x9F\x2A\xAB\x62\x41\xB3\x22\x67\xB0\x0B\xBA\x00\xB2\x12\x81\xB7\x6E\x10\xCC\x40\xB1\xC6\x88\xB6\xC5\xB2\xCB\x9C\xDC\xC6\x9E\x59\xA8\x60\xCB\x40\x01\x4C\x45\xDC\x00\x61\xA6\xC4\x9A\x8B\x51\x9F\xDF\x7D\x80\x03\x85\x85\xDB\xDB\xD0\x96\xD2\xDC\xA7\x04\x6E\x0B\x46\xE2\x02\x50\x12\x1F\xC4\x67\x0A\xA5\x77\xD0\x3D\xF2\x1E\xDD\xD8\x63\xC2\xE7\x49\xD3\x4F\xD1\xD6\x45\x59\x46\x86\xEB\x96\x8C\x21\x1B\x45\xEA\x40\x52\x2F\x1F\x1A\x52\x2B\xC6\x7E\xD5\x88\x5B\xAE\xCF\x62\xB0\x03\x68\xD4\xC6\xA4\x0A\x9E\x4F\x19\x1D\x4E\xE1\x12\x1D\x1E\x84\xC9\x44\xEE\x40\xB6\xD1\xAC\xA8\xCC\x42\x3B\xED\xEF\xEF\xC1\xC2\x41\xB1\xC4\x6C\xB7\xE7\xF1\xEE\x0F\x8A\xF3\x94\xF3\x46\x83\xF0\x01\xF1\x32\x96\xD9\xF0\xEF\xD7\xC0\xF9\xF4\xF4\x00\x0F\xA2\xF6\xC7\xE8\xDF\xFB\xF2\xF8\x38\x5A\xFC\xF4\xEE\x48\x95\xF0\xFA\xEF\xEC\xCF\x6C\x03\x46\xF7\xE1\xA5\xD2\xF9\x72\x1B\x45\x42\x91\xF9\x68\x21\x00\xC3\x3E\x49\xA4\x3C\x7E\x7F\xCD\x65\x22\xFB\x12\x69\x6D\xB5\x4E\x85\x7C\x9E\x74\x0E\x7D\x09\x0B\x22\x0A\xA5\x63\x4C\xA1\x23\x00\xF9\x43\x51\x2F\xB9\x35\x2E\x36\x04\x37\x61\x08\x92\x65\x38\xAA\x04\x00\x7E\xD2\x5F\x62\x45\x22\x83\x20\x16\x82\x27\x62\xE3\x49\x36\xF6\x6F\x2C\x20\x1A\x80\x06\x3F\x98\x61\x72\x81\x06\x0B\x23\x13\x8C\x26\x87\x00\x0E\x08\x53\x7E\x70\x88", "\x82\x00\x8F\x28\x49\x32\x45\x70\x08\x26\x82\x26\x83\x21\x6F\x18\x85\x0D\x11\x83\x20\x89\x7A\x54\x25\x28\x4E\x8C\x21\x13\xD0\x87\x65\x05\x82\x27\x50\x89\x45\x0E\x0D\x9C\x87\x6D\x0E\x94\x4E\x85\x00\x17\x60\xD4\x4E\x46\x23\x26\xA3\x87\x81\xC3\x25\x87\x0A\xEA\x60\x01\x35\xA2\x51\x80\x30\x70\x67\xE1\x74\x08\x8D\x8B\x70\x0F\x71\x6D\x47\x0F\x1C\x13\x08\x85\x13\x56\x23\x8C\x03\x36\x87\xE3\x2B\x71\x86\xAA\x14\x04\x8E\x6F\x80\x01\x23\x96\x28\x8A\x82\x18\x8C\x2E\x83\x5B\x6E\x97\x11\x91\x93\x49\xB4\x08\x91\x02\x36\x84\x49\x69\x58\x60\xAE\x4E\x00\x92\x2D\x83\x22\x27\xD3\x82\x21\x2A\xB7\x4B\x89\x9C\x89\x97\xAC\x40\x66\x6B\x39\x39\x62\x88\x64\x21\x50\x42\x20\x90\x00\x1B\xB8\x80\x70\x64\x75\x93\x40\x17\x9B\x32\xBD\x61\x33\x0E\x1F\x5B\x94\x5C\x02\x2F\x96\xB1\x7A\x86\x0D\xAC\x80\x02\x2B\x94\x2B\x3F\x1F\xBD\x81\x59\x45\x7D\x8F\x40\x76\x84\x2A\xFB\x3A\x94\x20\x31\x00\x52\xFF\x63\x9F\x31\x7F\x26\x9F\x9C\xE8\x89\x9C\x39\xE5\x99\x9D\x76\xA7\x9B\x9D\xB7\x2E\x9E\x3C\xF0\x90\x5A\x79\xAE\x9F\x9B\x03\x3A\x95\x12\x4E\x03\x0A\x2A\x12\x0D\x0A\x43\x14\x09\x12\x4F\x0E\x09\x2F\x0F\x08\x0A\x45\x0E\x09\x34\xC0\x8F\x04\x1E\x2F\x07\x07\xFE\x75\x00\x7F\x0C\x25\x9B\x1B\xBC\x9E\x9F\x00\xA2\xA0\x41\xDF\x03\x08\x26\x0F\x0B\x0A\x45\x0B\xA3\x40\x0F\xAF\x04\x1F\x32\x46\x02\x1B\x2F\x90\x43\x2A\xA5\x7D\x27\x0D\x08\x0A\x4C\x05\x08\x16\x5F\x05\x09\x98\x8D\x09\xA1\x54\x06\xA0\x42\x8A\xA7\x9A\x81\x0D\xA7\x05\x2F\x00\x87\x40\x17\x09\xA5\xA2\xB7\x63\x9F\x47\x11\xA4\x4D\xB6\xA8\xA6\x8E\x9E\xA0\xA4\x22\xBE\xA4\x50\xBE\x0A\x83\x14\x43\x20\x8F\x57\xAA\x94\x53\xCE\x09\xA7\x83\x89\xA2\xA8\x02\x36\x35\x43\xD1\x9A\x46\x84\x30\x96\x37\x5A\xA5\x0B\x4D\xDC\xAE\xA9\x8F\x85\x08\xAC\x00\x02\xAE\x0F\x5A\x74\x65\x46\x64\x7D\x08\x51\x15\x09\x10\x4C\x00\xAE\xB1\xBD\x04\x87\xEC\x89\x9D\x54\xC1\x81\x09\x9B\x95\xA4\x20\x15\xBB\x09\x18\x2D\x0A\x0F\x20\x2D\x02\x0B\x30\x0D\x05\x0E\x3A\x0E\x05\x2F\x2F\x05\x05\x5D\x0B\x04\x1A\x9B\x05\x91\x27\x94\x0C\x0A\x56\x1B\xAD\x15\x45\x04\x90\x86\x25\xB0\x00\x22\x1B\x0A\x17\x3C\x0E\x06\x11\x1D\x0A\x05\x22\x1C\x0F\x09\x2A\xBC\xB5\x13\x2F\xB7\x04\xF6\x20\x54\x42\x5A\x92\x38\x97\xAB\xAC\x0A\x22\x1B\x21\x70\xBE\xAE\xA1\x10\x2D\x05\x05\x28\x0A\xB7\x65\xBE\x0D\x0A\x3E\x2D\x02\xB9\x2D\x0C\xBA\x73\xCF\xBE\x0A\xE6\xA9\x02\x05\xC7\xA5\xA6\x1B\x83\x2D\x03\x0D\xB5\x75\x83\x41\x14\x09\x10\x2C\x0B\x22\xF2\x84\xBF\x05\x21\x0F\x07\x16\x72\x21\xBD\x20\x2A\xB9\xB9\x5E\x0F\x05\x17\x4D\xB2\xBE\xE5\xB3\xBC\x0F\xF2\xB5\xBD\x75\xDD\x07\xBB\xFC\x9D\x0D\xA4\xC2\x8E\x7A\x3B\x83\xB4\x0A\x99\x94\x0C\x04\x1B\x28\xC1\xB6\x34\xBE\x07\x1E\x22\x07\x04\x20\x0F\x06\x1C\x2F\x0E\x0D\x17\x35\x2C\xB1\x8E\xB0\xB2\x64\x94\xB6\xB2\xCC\xA9\x03\x05\x7C\x11\xB7\x09\x3C\x92\x9E\x19\x5E\x88\x00\x53\x10\x09\x10\x43\x05\x09\xAA\x88\x26\xC5\x2F\x13\x0F\x0A\x58\xA2\xC1\x76\xA1\x9C\x21\x01\xC0\x00\x16\xD8\x0C\x8A\x32\x6C\xA4\x20\x2C\xBC\xC5\x31\xD1\x8E\x05\xD4\x6B\x73\x0A\x75\x32\x0C\x1D\x1B\x9D\x4B\x81\x00\x08\x04\x24\x1C\x06\x90\x99\x91\x34\x1F\x39\xC3\xC7\x3D\xC1\x34\x3E\x82\x2A\x97\x29\xC6\x9D\x29\x48\xDE\x74\x1D\x27\x9C\x20\x32\xC0\x07\xC9\x52\xD9\xC9\x19\x56\xC4\x29\x81\x1E\xC9\x8A\x34\xC0\x01\x00\xEF\xC7\x64\x17\x31\x04\x07\x1B\x38\xCE\x40\x69\xC3\xC9\x93\x7E\x01\x5C\x4E\x05\xB1\x11\x20\x01\x5C\x2B\xA5\x0F\xC9\x22\x0E\xC0\x08\x61\xB0\xB4\xD1\x83\xD1\x5C\x60\x92\x70\x48\x0B\xDD\xCE\x32\x57\xCE\xCD\x6A\xD1\xCF\x9C\xEA\xCE\x04\x19\x3A\xC8\x21\x9F\xDC\xCC\x95\xEB\xC0\xD0\x41\xC4\xD7\x25\xB6\x49\xD2\x08\x4F\xCC\xD1\xD0\x8C\x0B\xB4\xA7\xD1\xD2\x48\x13\xDA\xCD\x4B\xE0\x41\xD3\xF8\x78\xD1\xA5\xA1\x3E\x05\x19\x1F\xD4\x20\xA1\xD4\xCC\x99\x7F\xC0\x05\x40\xF7\xA7\xD4\x86\xC0\x06\xAA\x8B\xDF\xB2\x57\xF0\xD0\xD2\x96\x53\xD7\x49\x6A\xC0\x08\x5B\xC0\x02\xD8\xA1\x29\x98\x87\x3A\xD3\xD5\x22\xEA\x74\x0E\x45\x0E\x68\x93\x82\x2E\x4B\x28\xD8\xDB\xCD\x20\x00\x0A\xD5\x58\x0E\x92\x6B\xEC\xDF\x5D\x0F\x72\x06\x03\x68\x33\xDF\x3D\xEF\xD8\x68\xC4\xC2\x04\x0F\x22\x05\xD8\x53\xC5\xD7\x4E\x4F\xDB\x0F\x4A\x6A\xC7\xD4\x4C\x9F\xCF\x24\x7D\x00\x04\x0A\x4A\xD5\xA4\x7E\xFC\x0A\xD9\xAD\xA0\x06\xC0\xD8\x08\x66\x31\x38\x92\x24\xE1\x56\x96\xC5\xF3\x30\x05\x84\xFA\xDF\x05\xFC\xC6\xDB\xA3\xDC\x24\x94\x76\x5C\x68\xE2\xFC\xC9\x04\x08\x13\xDE\x93\x06", "\x0A\x02\xCA\x9E\x99\xDF\xBE\xFD\xD7\xD9\x7F\xD4\x21\xDD\x13\xE4\xE1\xB8\x06\xE1\x8A\x84\xCA\xE4\xE1\x11\xEF\xE2\x08\x11\xE3\xE3\x8A\xD7\xE7\xD4\x1A\xF7\xE2\xCF\x9F\xE1\xE4\x7F\xE4\xE2\x5D\x26\xE9\xD5\xC4\xAA\xEC\xE4\x78\xDA\xD6\xCD\xCD\x99\x72\x81\x66\xC4\x20\x33\xD8\x09\xEB\xCF\x9D\xD0\x29\x1B\x23\xEC\x7B\xE0\xEC\x6C\x32\xFF\x05\xD3\xB5\xE8\xD4\x8A\x01\xE3\xE0\xBB\xD8\xE3\xCA\xF1\xEB\xE9\x1F\x11\xE3\xD2\x67\xE2\x72\xCB\xD2\xC9\xEE\x03\x69\xEB\xED\x23\xE9\xD8\x10\x39\xE1\xEF\x82\xF4\xED\xE3\xCA\xD6\xEC\xBF\xF8\xEF\x6E\x67\xA4\x6B\xDD\xED\xCD\xCC\x88\x1E\x9E\x04\x1A\x35\xD8\x21\x97\xF0\xE4\xBC\xB2\xE0\xF0\x64\xF6\x54\xF0\xA3\xC6\xF3\xA8\xBE\xEB\xE0\x11\x0D\xE0\x04\x42\xED\xB7\xDB\x94\xE7\x96\xA6\x08\xED\xF5\x64\x28\xF0\xD3\xB4\xE3\xE5\x51\x90\xEA\x63\x37\xE3\xE9\xCA\xAB\xE7\x25\x96\xEF\xE7\xDE\xA9\x3C\xF0\xED\x9E\xF7\xE6\xD0\xDF\xCA\xF4\x07\xE8\xF1\xE9\xA7\xF9\xF4\x7E\xC3\xF3\x25\x46\xEE\xF5\xC6\xB0\xF8\x64\x8F\xCE\xA5\xF3\xE1\x56\xF5\x29\xD2\xEB\xF7\xAA\xF4\x91\x4B\xC0\xF4\x2A\xBB\x94\xF3\x06\x43\x1B\x26\xFC\xF6\xD1\xE5\xB8\x7A\xD8\xFB\xB6\xE2\x26\xF8\x51\x88\xF9\xCF\xCA\xF0\xE8\x87\xE8\xE2\xE2\xD0\xF4\xFB\xEB\x9B\xE1\xF6\xEB\xE0\xE3\xF8\xD9\x7D\x26\xB7\x77\x72\xBA\x70\x02\x54\x7D\x7B\xF0\x60\x04\x57\x70\x22\x7D\x7E\x06\x5C\x73\x10\x5E\x7D\x80\xBD\x61\x05\x1B\x14\x81\x66\x7D\x78\x5A\x7E\x06\x7F\x70\x80\xEE\x78\x7F\xE7\x62\x7F\x27\x35\x77\x77\x77\x12\x0D\x8A\x6E\xE0\x71\x79\x38\x6E\x66\x94\x76\x03\xE8\x78\x10\x2D\x8A\x79\xF8\x6B\x7E\x33\x72\x72\xC4\x7F\x7E\x70\x71\x7A\x3B\x77\x78\x09\x7B\x7C\x41\x77\x7F\x45\x7A\x7F\xD3\x7C\x7F\xD5\x7A\x6C\xD7\x7F\x64\x34\x81\x80\x25\x78\x7B\x22\x15\x80\x07\x82\x2E\xF0\x67\x62\x58\x08\x44\x7C\x44\x47\x51\x85\x34\x76\x68\x05\x1B\x18\x85\xF6\x62\x85\xC4\x67\x84\xA6\x2C\x46\x4A\x1F\x70\x78\x5A\x57\x2E\x19\x7D\x3C\x2F\x24\x0F\x7D\x6A\x8E\x67\x6A\xDC\x44\x22\x0F\x75\x60\x51\x8C\x86\x67\x84\x22\x93\x62\x85\x40\x00\x58\x08\x18\x87\x58\x02\x17\x54\x2B\x6E\x7C\x85\x17\x76\x63\x02\x1B\x12\x88\xF6\x6F\x87\x61\x04\x6C\xEC\x52\x5E\x72\x80\x86\xBC\x70\x02\x86\x86\x82\xAC\x2B\x87\x11\x4E\x6E\x71\x49\x1A\x76\x69\x02\x1B\x18\x89\x85\x81\x41\x88\x8E\x5B\x38\x5B\x88\xA2\x2D\x88\x95\x84\x15\x90\x88\x3E\xE1\x74\x49\x92\x77\x6D\x94\x71\x03\x2D\x84\x10\x2F\x87\x8A\x74\x0C\x79\x46\x8E\x74\xC9\x60\x7F\x27\x30\x82\x7B\x34\x7F\xA6\x7D\x83\x05\x8F\x83\x42\x8B\x7F\x4A\x72\x7B\xD7\x7C\x81\xDA\x71\x75\xB9\x73\x75\x8D\x8E\x49\x90\x70\x6F\x5B\x60\x00\x5D\x64\x79\x46\x60\x6E\x4E\x37\x56\x02\x1B\x44\xE6\x61\x73\xE9\x6B\x6E\xD1\x72\x13\xEE\x6B\x8D\x4D\x11\x03\x35\x0B\x11\xE1\x86\x6F\xDE\x84\x6C\x21\x05\x85\x44\x2F\x70\xE7\x86\x88\xA8\x70\x02\xEB\x81\x41\xED\x87\x8E\x9E\x40\x04\xDB\x52\x10\x4D\x48\x05\x7B\x4C\x4B\x36\x48\x8F\xB1\x31\x03\x60\x0B\x11\xFF\x86\x6F\x7B\x44\x6C\x13\x7D\x80\xA7\x6F\x47\x90\x37\x12\x0F\x72\x89\x72\x4A\x14\xE1\x28\x90\x81\x4E\x73\xB0\x87\x6A\x8B\x40\x6E\xF0\x6D\x88\x7B\x4A\x80\xB9\x6C\x80\x18\x8F\x4C\x3F\x63\x10\x41\x62\x81\xD1\x81\x6E\x64\x07\x58\x03\x15\x51\xE6\x6D\x80\x3E\x09\x6E\xF5\x80\x00\xF7\x89\x8F\xCB\x47\x6D\x7B\x4D\x69\x01\x98\x10\x35\x92\x65\x03\x91\x2E\xCF\x7F\x75\x1D\x94\x36\x07\x90\x48\x6F\x02\x48\xED\x8C\x90\xC6\x1F\x90\x40\x92\x94\x09\x73\x91\xE1\x25\x91\x9E\x48\x91\x5B\x2B\x8C\x40\x33\x7E\xF0\x6E\x02\x37\x0D\x8A\x03\x1F\x8A\xEA\x74\x84\xED\x75\x8B\x37\x87\x7C\x39\x83\x7A\xBA\x8C\x7C\x3E\x89\x7F\xBF\x81\x84\xC1\x8E\x7F\xED\x75\x8C\x4A\x88\x68\xC8\x8E\x7D\x32\x44\x3E\xE8\x84\x07\xD4\x80\x00\x4B\x41\x97\xDE\x6C\x48\x4E\x36\x6D\xE6\x6C\x85\xE9\x68\x87\x04\x1A\x87\x86\x8F\x80\x02\x1E\x75\x86\x87\x97\xD2\x84\x06\xFA\x83\x10\xE6\x66\x88\x2B\x95\x8A\x30\x34\x94\x54\x1D\x8C\x5D\x63\x8A\x2D\x33\x92\xD3\x8D\x1D\x00\x0B\x98\x9C\x89\x8D\xE0\x7E\x8D\x92\x9E\x66\xDE\x86\x98\x24\x93\x97\xD6\x82\x65\xE5\x8C\x92\x15\x4E\x66\x7B\x46\x92\x02\x15\x51\xFC\x8A\x21\xA2\x9E\x34\x82\x90\x00\x66\x66\x6E\x03\x99\x6E\x0B\x87\x6D\x0D\x83\x9B\x11\x8D\x93\x82\x36\x67\x16\x88\x10\xC1\x9C\x77\xBE\x9E\x77\x32\x8A\x76\xB3\x8C\x76\x5C\x9F\x81\x5F\x99\x7C\x1E\x7A\x78\x8D\x8D\x80\x50\x82\x85\xE3\x60\x00\x5E\x26\x97\xD1\x81\x66\x4A\x66\x6D\x4F\x6B\x97\x45\x33\x57\x70\x92\x85\x89\x90\x22\x52\x8E\x02\xDA\x94\x07\x2D\x9D\x44", "\x68\x61\x97\x3E\x0E\x97\x03\x10\x98\x1E\x41\x06\x9F\x90\x22\x85\x99\x9D\x49\x6F\x49\x31\x9D\x9D\x52\x6C\x98\xEE\x92\x10\xF0\x94\x25\xB9\x9B\x87\xF1\x96\x9E\xF7\x9C\x9D\x20\x0B\x9F\xF1\x9E\x03\x7E\x81\x41\xDA\x63\x10\xC9\x40\x99\x95\x9F\x6D\xE7\x94\x9A\x4E\x60\x65\xFB\x9C\x89\x0B\xAD\x90\xBC\x9E\x66\x94\x9F\x30\xE7\x99\x9B\xEB\x9D\xA1\x0A\xA7\x82\xF0\x6D\xA0\x02\x1F\xA0\xD0\x82\xA1\xF7\x94\xA1\x02\x1A\x9F\xB0\x83\xA2\x91\x8E\x49\x1B\xA3\x79\x53\x9F\xA1\x1B\x97\x66\x07\xAF\xA2\xDA\x84\x7D\x26\xA0\x00\x0F\xA1\x9A\xF6\x92\x66\x2C\xA8\xA3\xE6\x65\x8E\x3B\xA2\x84\x33\xA7\x6D\xA1\x96\xA3\x0C\x11\x40\xEB\x95\x8E\x01\xAD\x80\x28\x5C\x10\x53\xAA\x9B\xC5\x95\x36\x41\xAA\x64\xAC\x90\x00\x15\x5E\xA2\x2A\x91\xA5\x57\xAD\xA3\x0F\xA2\x81\xE7\x93\x9F\x38\xA4\x9C\x39\x48\x76\xE9\x9A\x9A\x5B\x27\x5A\x04\x17\x5A\xAF\x93\x19\x86\x97\x9E\x49\xAE\xA2\x03\x9B\xA6\x20\x2B\x47\xD5\x97\x9D\x33\x93\xA1\x98\x9B\x9E\x03\x9E\x6E\xB8\x6D\xA4\xDC\x6F\x65\x95\x7F\x6B\x03\x11\x6C\x6A\x6D\x85\xB4\x89\x6A\x8A\x6A\x86\xAF\x63\x72\xB2\x6D\x88\xD4\x63\x9D\xD9\x44\x85\xE5\x92\x03\x5A\x88\x10\x9B\xA2\x65\x5C\x85\x6A\x6C\x7D\x86\x61\x82\x2E\x79\x5B\x57\xA0\x88\x86\xCC\x6D\x68\x91\xA7\xAA\x6E\x82\x2F\x06\x6C\xAA\x4A\x15\x87\x45\x37\x87\x1B\x1F\x9F\x75\x1E\x87\x03\xA2\x03\x84\x88\x10\xBB\xA8\xA0\x54\x28\x88\x44\x0D\x5E\xB0\xA2\x8A\xF1\x9E\x98\xD4\x30\xA1\x74\x04\x89\x11\x4D\x69\x9A\x88\x10\xCD\xA7\xA1\x0D\x9D\x89\x4D\x0F\x5B\xC3\xA3\x69\x1D\xA6\xAC\xF1\x30\x8B\x52\x90\x34\x2E\x05\x03\x56\x92\x10\x58\x9B\x79\x32\x8B\x95\x9F\x7D\x95\xF1\x7D\x9C\xF3\x7B\x83\xF5\x7D\x8E\xBD\x84\x96\x47\x76\x96\x1C\x78\x96\xC4\x82\x80\xDB\x77\x8C\xDD\x78\x80\x40\x3A\x91\x84\xAC\x91\x69\xA2\x38\x49\xAC\x93\xFD\xA6\x20\x95\x71\x9C\x04\x13\x9C\x00\xB0\x4C\x82\x3B\x81\x5E\x8B\x9C\x3A\x73\x77\x60\x9F\x9C\x23\x80\x02\x25\x88\xA7\x32\x9B\x25\xE3\x95\xB1\xB0\x93\x03\x35\x94\x10\x37\x91\xA7\xC1\x71\x2E\x05\x97\xA5\x3F\x99\x90\x43\x9D\xA1\x23\xB1\x91\x88\x7A\x94\x20\x0C\x94\x17\x93\x69\x19\x96\x1F\x82\xA1\x77\x14\xBA\x66\xF4\x49\x10\xEE\x46\xA9\x49\x14\x60\x45\x3E\x3F\xBA\x55\x4F\xED\x4D\x9F\x00\x06\xAB\x61\x07\x62\x86\x8E\x3F\x77\x3E\xB3\xE9\x49\xA1\xA9\x1C\x5B\x10\x4D\x90\xFE\x30\x02\x38\x1E\x4E\x50\x9B\xB4\x74\x4E\x49\xFE\x33\x5C\xF1\x4D\x4E\x47\xA7\x71\x83\x57\x71\xFE\x3A\x60\x59\xB9\x4E\x60\xA1\xB0\x6E\x07\xB1\x0D\x8E\x3F\xB2\x21\xB6\xE8\x44\xB1\x1E\xBD\x4D\x71\x3B\x25\xFE\x38\x02\x51\xBD\x4E\x10\x0E\x62\x09\x16\x24\xFE\x32\x23\x35\xBB\xB7\x7C\xB7\x1B\x46\x2C\x4B\x70\x5C\xB7\x32\x2A\xB7\x3D\x58\x10\xD3\x45\x06\xFE\x31\xB3\xA3\x65\xB3\x09\x4F\x0F\x39\x66\xB7\x7D\xB1\xB9\xE3\x4D\xB8\x71\x61\x4D\x82\xBB\x11\x84\xB5\x46\x9A\xB0\x00\xB2\x69\xB8\xA9\x87\x6E\xD0\x26\x24\x26\x62\xB9\x04\x19\xB9\xF4\x43\xB3\x71\x7E\xB9\x29\x87\x6B\x71\x74\x35\xCE\x30\x57\x68\x49\xB7\xA8\x42\x10\x90\xB1\xB9\xA8\xBB\xB8\xA9\x86\x49\x85\xA1\x45\x40\xB1\xBB\x7B\xBA\xB8\x5F\x61\xB8\xB9\xB3\x4C\x03\x1C\x73\x21\x8E\x3F\x63\xB8\xB0\x06\x2E\xBA\x41\xBB\xB7\x32\x20\xB4\xB5\xBD\xB7\xB7\xBF\x65\xAA\xB2\x14\xBB\xB7\x32\x40\xB0\xB4\x7C\xB0\xBC\x51\x42\xBC\xAB\xB6\xBD\x02\x16\xBC\x7B\x3E\x3F\x53\xB4\x15\xA4\xBC\xB7\x8D\xB8\x46\x2D\x97\xBE\xA1\xB8\x13\x2D\x96\x1F\xEC\xB7\xB7\x38\x10\xB8\xF1\xB0\x57\xDA\x3A\xBD\x7B\xB5\xBE\x88\x3B\x4E\x83\x56\x68\xD5\xBA\xBB\xAC\xB8\xBB\x38\x13\x3E\xDF\xB1\xBF\xE4\x4D\xB8\xF4\xBC\xBE\xF6\xB3\xBB\x7D\xB9\xBC\x8E\x72\x38\xA6\xB6\x44\x85\xBE\xBF\xC3\xB2\x10\x58\x69\xBA\x93\xB6\x24\x40\xBC\x4B\x06\xCF\x3F\x2F\x3D\x64\x0A\xCE\x5D\xF1\x34\xBA\xFA\xB4\x07\xDA\x35\x02\xF1\xBB\xB5\x43\x4D\xB8\x39\x61\x67\x7C\xB1\xBD\xDA\x30\xBF\xF9\xB4\xA2\x40\x37\xC2\x29\xCD\x99\xD4\x71\xC1\xB2\x4B\x5B\xDB\x43\x87\x74\x0E\x3F\x38\xB4\x07\x08\xC7\xA3\xA4\xB0\xC3\x28\x4C\xB7\x25\xC6\xC3\xEC\xBA\xC2\x32\x1A\xC3\x2F\x44\x47\xA2\xA0\xC4\xE1\x95\x34\x94\xB4\xC4\x92\xB6\xC4\xD3\x31\xB9\x49\xCB\x11\x28\xCB\xC4\x38\xC2\x84\x4E\xC9\x46\x50\xCE\xC3\x52\xCB\xB9\x71\x9D\xB8\x73\x93\x9B\x2F\xC2\xC2\x29\x47\xBE\x5B\xCC\x10\x5D\xCE\xC6\x5F\xC7\x71\x61\xCC\xB4\x51\xCD\x59\xF4\x91\x41\x8D\xB3\x9B\x73\x9B\xC6\x9B\xB9\x47\x92\xBF\xC6\x08\x11\xC7\x24\xC3\xC7\x4C\x15\xC7\x63\xC0\x86\xFE\x39\xBD\x1C\xA1\x9F\x8D\xB5\x9D\x2E\xCB\xB7\x58\xC8\x3E\x85", "\xC1\xA3\x4F\x8C\xC5\x37\xC1\x89\xDE\x88\xC8\x0F\x47\xC7\x7F\xCF\x5D\xE4\x9B\xB3\x8D\xB2\x23\xCD\x87\xB9\xEC\xBC\xB4\x56\xB8\xC7\xAA\x80\x6F\xFF\x01\xBD\xCC\xBD\xA3\xB2\xBC\x10\xCD\xA4\x10\xD1\xBC\xB7\xA8\xC0\x6F\xE4\xB4\xC3\xA9\x10\xB4\xB3\xC1\xC8\xBA\xC3\x37\xE4\x47\xB1\x2C\xB0\x34\xFE\x3C\xB6\x7B\x4C\xBC\x5B\xA1\xCB\x83\xCB\x11\xB5\xCB\xB7\xB7\xC4\xCC\xAA\xC0\x8B\x40\xB4\xC8\xBE\xC6\xC9\x40\x3D\xCC\x5E\xCA\xC9\xD4\x7F\xCA\xCE\xB6\x54\x2F\x61\xBF\xCF\xC9\x1A\xB9\xC6\xCD\xBB\xCE\xCD\x95\xC6\x8A\x9E\x41\xC6\xC2\xCF\x6D\xA9\xCA\x45\xA9\x99\xA7\x70\xB4\x4F\xA3\xCB\x11\x6E\xA2\xB9\x61\xCE\x3F\x66\xB7\xA5\xC8\xB7\xA3\x07\xBF\xC0\xCB\xB0\x57\xCC\xBF\x91\xE4\x42\x23\xBD\xC4\xBB\xF1\xB9\xCF\x64\xBB\xCF\xFB\xA6\xA5\x64\xBC\xCB\xF1\xBE\xC0\x19\x8E\xBD\x3C\xCF\x40\x10\x91\x94\x73\x05\xCC\xEF\xC8\xB1\x93\x11\xD0\xDD\xCB\x52\xCC\xC7\xD0\x31\x97\xB6\x56\xC2\x10\x73\x6D\x80\x40\xB2\xD0\x0D\xCC\xCF\x0D\x82\xD1\xB1\x4F\xC4\x02\x15\xD1\x82\x48\xD1\xEE\xC0\xCF\x1A\x26\xC0\x43\xCC\x10\x12\x5C\xBE\x83\xBB\xB7\x17\xBA\x96\x5C\x33\xCE\xE8\xC8\xCB\xED\xBF\x18\x27\x51\xBF\xF7\xC5\xB8\x12\xC6\x14\x3B\xBC\xC6\x8C\x27\xC6\x46\x20\x57\xC9\xC8\xB9\x98\xCC\x10\xD8\xC4\x4F\x3A\xB9\x14\x65\xC6\x57\xA1\xC9\x14\xD3\xC1\xBF\x41\xC5\xD5\xE7\xCF\x98\xDE\x8C\xB8\xA2\xBB\x11\x44\x5A\xD3\x5E\xD4\x3E\x45\xB5\xAB\x6C\xD4\x9F\x8E\xCF\xD4\x13\x54\x4F\x32\x24\xCD\x06\xDC\xBE\x44\xB1\x9F\x8B\xCD\xD6\xD7\x66\x88\x5D\xDC\xBE\x8C\xCE\xD6\x54\x20\xD6\xD5\xC6\x8A\x63\xD2\xCF\x65\xD6\x55\x91\xB7\xD4\x0F\x41\x49\x1D\xA0\xD4\x8F\x9D\xA1\xAD\xC8\x13\x70\x58\xD2\xCA\xC4\x10\x05\xD0\x00\x82\xD4\x4E\x4C\xBD\xD8\xD1\xCD\xA1\x0E\xD9\xCD\xA6\x8D\xA1\x2C\xDB\xC3\x91\x40\xD3\x17\xD9\xD1\x04\x1F\x92\x5B\x28\xCC\x1D\xDB\xCC\x54\xD0\xCE\x74\x4C\xD9\x59\xCD\xC6\xC8\xAD\xD7\x72\xC1\x89\x1D\xA9\xD9\x7D\xBC\xC4\x78\x23\xD9\xAD\xD0\x64\x1F\xD7\xCA\xB1\xD1\x41\x8E\xD7\xAC\x9E\xD6\xCE\x83\xDF\xD8\x11\x43\xDA\x2E\xD3\xD1\x00\x06\xDA\x32\xD9\x45\x1A\xD2\xCE\xF1\xB7\xD3\x79\x86\xD4\x52\xD5\xB3\xFD\xB7\x27\xC5\xD9\xAD\xED\xC1\xB9\xF3\xCC\x10\x65\x5D\xC1\xF2\xBF\x18\x70\x52\xC3\x0C\x1D\x3D\x1B\x10\x57\x40\xB8\xDE\xF4\x4C\xDB\xCC\xDD\xC6\x30\x39\x74\xF3\x1D\xDD\xB4\xDF\xDD\x07\xC7\xD8\x08\x15\x57\xA4\xBA\xD8\x94\xC4\x3D\x7B\x41\xDD\xA9\xDC\xA6\x35\xD4\xDE\x03\xC8\x13\xD1\x4C\x59\x68\xDF\xC9\x02\x14\xD9\x7B\xB6\xCC\xF1\xCE\xC1\x1D\x34\x10\xD5\x96\xCA\x5A\xD8\x10\x0E\xE5\xB3\x10\xE5\xE0\xCE\x36\x24\x69\xC5\xBF\x6C\xC9\xE1\xA7\xB9\xD1\x6D\xBB\xBF\x9A\xD9\xC8\xA2\x26\xDF\x3B\x18\xAC\xEB\x4D\xB8\x14\xB6\xE0\x30\xE1\xE3\x32\xED\xE2\x7F\xBC\xC0\x16\xE5\xBA\x7B\xBB\xDD\x14\x1A\xD0\x02\x48\x75\xFA\xC5\xDE\xB7\x1E\x3F\xDD\x53\xE3\x36\xD6\x24\x2D\xC8\xDD\xE9\xDE\xCC\x28\xE6\xBE\x4C\xDE\x66\x52\x86\xDD\x04\x11\xC9\x35\xBB\xD3\x35\xB6\xC7\x64\xCD\xE4\x5B\xD2\xC4\x00\x43\xD2\x7D\xB4\xE5\xF4\x46\xE5\x8A\xC8\xE5\xD7\x62\x85\x1D\xE8\x13\x90\xC8\xE4\x08\x1C\xC9\xCE\xDE\xC9\x17\xE4\x46\x63\xE4\xC5\x78\xBB\x11\x1F\x04\xE4\x30\xEB\xC7\x98\x9D\xDF\xD9\xDF\xE5\x4B\xE6\x11\x79\xDC\x10\x42\xB2\xE1\x41\xE6\x3A\x74\xE4\xDE\x68\xCB\x11\x6A\xCF\xDB\x03\x1A\xE6\x91\x4C\xE6\x80\xC9\xAD\x9C\xDD\xB8\xFE\x3B\xAB\x83\xED\xB7\x8D\xB6\x1F\x15\xE2\xE3\x90\xE3\xD5\x93\xE4\xE9\xA1\x37\x19\x68\xEE\xE6\x04\x14\xD7\xBB\xD6\xC8\x32\x10\xE8\xD0\x27\xE4\x21\xCC\xE0\x00\x02\xEA\x48\xC4\xEA\x65\x0A\xE8\x3B\xCC\xE8\xA0\xC7\xE1\x4E\xD3\xBF\x5C\xEC\xB7\x0B\xC0\xC7\x99\xC4\xD8\x39\xCA\xE4\x9D\xC7\xE5\xAA\xED\xD4\x4F\xEB\xE5\x0B\xD4\xBA\xB9\xEE\xDA\xB7\xDC\xEB\x60\xCE\xEB\x6B\xE0\xEC\x6D\xE3\xEB\xB5\xEF\x18\x1F\xE9\xEA\xCF\xE2\x10\xAC\xE3\xC3\xDA\xCB\xEC\x55\xEB\xE7\x3F\xCD\xD9\x7A\xC6\x24\x7C\xC0\xE2\xAA\xE7\xED\x35\xBC\xDB\xB0\xED\xDC\xB2\xEE\x9F\x7A\xD2\xA0\x54\x2F\xC8\xB7\xEB\xB7\xC7\xE1\xEA\xBB\xE2\xD6\xBD\xEB\xED\xBF\xE1\xE6\xC1\xEE\xE4\x70\xE8\x13\xFE\x37\x02\x9B\xE4\xD6\x08\xEB\x11\x0A\xEF\xEF\x40\xE0\x2D\xFE\x37\xD9\x04\xFE\xB7\x01\xFC\x10\x03\xF8\xE7\xAA\xE8\xF0\x0F\xE4\xE2\x01\xE5\xE6\x13\xE0\xC2\xE7\xB3\xBA\xB2\xC1\xBF\x1B\xE3\x19\x43\xD7\x1B\xD3\xE7\xBE\xF1\xE3\x10\x10\xFA\xE1\x12\xF3\x3A\xCC\xEB\xE8\xCE\xED\xE8\xB4\xD2\xDB\x09\xFD\xF2\x44\xE0", }; void XmlGetParserBuffer(vl::stream::MemoryStream& stream) { vl::stream::MemoryStream compressedStream; for (vint i = 0; i < parserBufferRows; i++) { vint size = i == parserBufferRows - 1 ? parserBufferRemain : parserBufferBlock; compressedStream.Write((void*)parserBuffer[i], size); } compressedStream.SeekFromBegin(0); vl::stream::LzwDecoder decoder; vl::stream::DecoderStream decoderStream(compressedStream, decoder); vl::collections::Array buffer(65536); while (true) { vl::vint size = decoderStream.Read(&buffer[0], 65536); if (size == 0) break; stream.Write(&buffer[0], size); } stream.SeekFromBegin(0); } /*********************************************************************** Unescaping Function Foward Declarations ***********************************************************************/ extern void XmlMergeTextFragment(vl::collections::List>& value, const vl::collections::List& tokens); extern void XmlUnescapeAttributeValue(vl::parsing::ParsingToken& value, const vl::collections::List& tokens); extern void XmlUnescapeCData(vl::parsing::ParsingToken& value, const vl::collections::List& tokens); extern void XmlUnescapeComment(vl::parsing::ParsingToken& value, const vl::collections::List& tokens); /*********************************************************************** Parsing Tree Conversion Driver Implementation ***********************************************************************/ class XmlTreeConverter : public vl::parsing::ParsingTreeConverter { public: using vl::parsing::ParsingTreeConverter::SetMember; void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->content, obj->GetMember(L"content"), tokens); } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { if(SetMember(tree->content, obj->GetMember(L"content"), tokens)) { XmlUnescapeCData(tree->content, tokens); } } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->name, obj->GetMember(L"name"), tokens); if(SetMember(tree->value, obj->GetMember(L"value"), tokens)) { XmlUnescapeAttributeValue(tree->value, tokens); } } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { if(SetMember(tree->content, obj->GetMember(L"content"), tokens)) { XmlUnescapeComment(tree->content, tokens); } } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->name, obj->GetMember(L"name"), tokens); SetMember(tree->closingName, obj->GetMember(L"closingName"), tokens); SetMember(tree->attributes, obj->GetMember(L"attributes"), tokens); if(SetMember(tree->subNodes, obj->GetMember(L"subNodes"), tokens)) { XmlMergeTextFragment(tree->subNodes, tokens); } } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->name, obj->GetMember(L"name"), tokens); SetMember(tree->attributes, obj->GetMember(L"attributes"), tokens); } void Fill(vl::Ptr tree, vl::Ptr obj, const TokenList& tokens) { SetMember(tree->prologs, obj->GetMember(L"prologs"), tokens); SetMember(tree->rootElement, obj->GetMember(L"rootElement"), tokens); } vl::Ptr ConvertClass(vl::Ptr obj, const TokenList& tokens)override { if(obj->GetType() == L"Text") { auto tree = vl::Ptr(new XmlText); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"CData") { auto tree = vl::Ptr(new XmlCData); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Attribute") { auto tree = vl::Ptr(new XmlAttribute); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Comment") { auto tree = vl::Ptr(new XmlComment); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Element") { auto tree = vl::Ptr(new XmlElement); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Instruction") { auto tree = vl::Ptr(new XmlInstruction); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else if(obj->GetType() == L"Document") { auto tree = vl::Ptr(new XmlDocument); vl::collections::CopyFrom(tree->creatorRules, obj->GetCreatorRules()); Fill(tree, obj, tokens); Fill(tree.Cast(), obj, tokens); return tree; } else return 0; } }; vl::Ptr XmlConvertParsingTreeNode(vl::Ptr node, const vl::collections::List& tokens) { XmlTreeConverter converter; vl::Ptr tree; converter.SetMember(tree, node, tokens); return tree; } /*********************************************************************** Parsing Tree Conversion Implementation ***********************************************************************/ vl::Ptr XmlText::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr XmlCData::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr XmlAttribute::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr XmlComment::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr XmlElement::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr XmlInstruction::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } vl::Ptr XmlDocument::Convert(vl::Ptr node, const vl::collections::List& tokens) { return XmlConvertParsingTreeNode(node, tokens).Cast(); } /*********************************************************************** Parser Function ***********************************************************************/ vl::Ptr XmlParseDocumentAsParsingTreeNode(const vl::WString& input, vl::Ptr table, vl::collections::List>& errors, vl::vint codeIndex) { vl::parsing::tabling::ParsingState state(input, table, codeIndex); state.Reset(L"XDocument"); vl::Ptr parser=vl::parsing::tabling::CreateStrictParser(table); vl::Ptr node=parser->Parse(state, errors); return node; } vl::Ptr XmlParseDocumentAsParsingTreeNode(const vl::WString& input, vl::Ptr table, vl::vint codeIndex) { vl::collections::List> errors; return XmlParseDocumentAsParsingTreeNode(input, table, errors, codeIndex); } vl::Ptr XmlParseDocument(const vl::WString& input, vl::Ptr table, vl::collections::List>& errors, vl::vint codeIndex) { vl::parsing::tabling::ParsingState state(input, table, codeIndex); state.Reset(L"XDocument"); vl::Ptr parser=vl::parsing::tabling::CreateStrictParser(table); vl::Ptr node=parser->Parse(state, errors); if(node && errors.Count()==0) { return XmlConvertParsingTreeNode(node, state.GetTokens()).Cast(); } return 0; } vl::Ptr XmlParseDocument(const vl::WString& input, vl::Ptr table, vl::vint codeIndex) { vl::collections::List> errors; return XmlParseDocument(input, table, errors, codeIndex); } vl::Ptr XmlParseElementAsParsingTreeNode(const vl::WString& input, vl::Ptr table, vl::collections::List>& errors, vl::vint codeIndex) { vl::parsing::tabling::ParsingState state(input, table, codeIndex); state.Reset(L"XElement"); vl::Ptr parser=vl::parsing::tabling::CreateStrictParser(table); vl::Ptr node=parser->Parse(state, errors); return node; } vl::Ptr XmlParseElementAsParsingTreeNode(const vl::WString& input, vl::Ptr table, vl::vint codeIndex) { vl::collections::List> errors; return XmlParseElementAsParsingTreeNode(input, table, errors, codeIndex); } vl::Ptr XmlParseElement(const vl::WString& input, vl::Ptr table, vl::collections::List>& errors, vl::vint codeIndex) { vl::parsing::tabling::ParsingState state(input, table, codeIndex); state.Reset(L"XElement"); vl::Ptr parser=vl::parsing::tabling::CreateStrictParser(table); vl::Ptr node=parser->Parse(state, errors); if(node && errors.Count()==0) { return XmlConvertParsingTreeNode(node, state.GetTokens()).Cast(); } return 0; } vl::Ptr XmlParseElement(const vl::WString& input, vl::Ptr table, vl::vint codeIndex) { vl::collections::List> errors; return XmlParseElement(input, table, errors, codeIndex); } /*********************************************************************** Table Generation ***********************************************************************/ vl::Ptr XmlLoadTable() { vl::stream::MemoryStream stream; XmlGetParserBuffer(stream); auto table = vl::Ptr(new vl::parsing::tabling::ParsingTable(stream)); table->Initialize(); return table; } } } }