Skip to content

Commit

Permalink
Implement parsing to a list of tokens (#695)
Browse files Browse the repository at this point in the history 
Implement parsing to a list of tokens

Added to TopLevelParser:
* def parseToTokens(input:RiddlParseInput): Either[Messages,List[Token]]

Added these case classes to AST as the set of possible results returned from TopLevelParser.parseToTokens:
* case class PunctuationTKN(at: At) extends Token
* case class QuotedStringTKN(at: At) extends Token
* case class ReadabilityTKN(at: At) extends Token
* case class PredefinedTKN(at: At) extends Token
* case class KeywordTKN(at: At) extends Token
* case class CommentTKN(at: At) extends Token
* case class LiteralStringTKN(at: At) extends Token
* case class MarkdownLinesTKN(at: At) extends Token
* case class IdentifierTKN(at: At) extends Token
* case class OtherTKN(at: At) extends Token

This allows for distinguishing the basic kinds of input being parsed.
  • Loading branch information
@reid-spencer
reid-spencer authored Nov 23, 2024
1 parent ef3af14 commit 2747020
Show file tree
Hide file tree
Showing 14 changed files with 699 additions and 90 deletions.
1 change: 1 addition & 0 deletions language/jvm/src/test/scala/com/ossuminc/riddl/language/parsing/JVMTests.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ class JVMRepositoryTest extends RepositoryTest
class JVMStatementsTest extends StatementsTest
class JVMStreamingParserTest extends StreamingParserTest
class JVMTypeParserTest extends TypeParserTest
class JVMTokenStreamParserTest extends TokenStreamParserTest
15 changes: 15 additions & 0 deletions language/jvm/src/test/scala/com/ossuminc/riddl/language/parsing/TopLevelParserTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,30 @@ class TopLevelParserTest extends ParsingTest {
"parse RiddlParserInput" in { (_: TestData) =>
TopLevelParser.parseInput(rpi) mustBe Right(simpleDomainResults)
}

"parse from a URL" in { (_: TestData) =>
val url: URL = PathUtils.urlFromCwdPath(Path.of("language/jvm/src/test/input/everything.riddl"))
val future = TopLevelParser.parseURL(url)
Await.result(future, 10.seconds) match
case Right(r: Root) =>
r.domains.head.id.value must be("Everything")
case Left(messages: Messages) =>
fail(messages.format)
end match
}

"parse File" in { (_: TestData) =>
TopLevelParser.parseInput(rpi) mustBe Right(simpleDomainResults)
}

"parse String" in { (_: TestData) =>
val source = Source.fromFile(simpleDomainFile.toFile)
try {
val result = TopLevelParser.parseInput(rpi)
result mustBe Right(simpleDomainResults)
} finally { source.close() }
}

"parse empty String" in { (_: TestData) =>
val parser = StringParser("")
parser.parseRoot match {
Expand Down Expand Up @@ -88,6 +102,7 @@ class TopLevelParserTest extends ParsingTest {
paths must contain("language/jvm/src/test/input/everything_full.riddl")
}
}

"return URLs on failure" in { (td: TestData) =>
val rpi: RiddlParserInput = RiddlParserInput("some source that ain't riddl", td)
val tlp = TopLevelParser(rpi, false)
Expand Down
16 changes: 16 additions & 0 deletions language/shared/src/main/scala/com/ossuminc/riddl/language/AST.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3959,6 +3959,22 @@ object AST:
override def format: String = s"domain ${pathId.format}"
}

/////////////////////////////////////////////////////////////////////////////////////////////////////////// TOKENS
sealed trait Token {
def at: At
}

case class PunctuationTKN(at: At) extends Token
case class QuotedStringTKN(at: At) extends Token
case class ReadabilityTKN(at: At) extends Token
case class PredefinedTKN(at: At) extends Token
case class KeywordTKN(at: At) extends Token
case class CommentTKN(at: At) extends Token
case class LiteralStringTKN(at: At) extends Token
case class MarkdownLinesTKN(at: At) extends Token
case class IdentifierTKN(at: At) extends Token
case class OtherTKN(at: At) extends Token

/////////////////////////////////////////////////////////////////////////////////////////////////////////// FUNCTIONS

/** Find the authors for some definition
Expand Down
6 changes: 3 additions & 3 deletions language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/CommonParser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ private[parsing] trait CommonParser(using io: PlatformContext)

def literalStrings[u: P]: P[Seq[LiteralString]] = { P(literalString.rep(1)) }

private def markdownLines[u: P]: P[Seq[LiteralString]] = {
def markdownLines[u: P]: P[Seq[LiteralString]] = {
P(markdownLine.rep(1))
}

Expand Down Expand Up @@ -228,12 +228,12 @@ private[parsing] trait CommonParser(using io: PlatformContext)
private def portNum[u: P]: P[String] = {
P(Index ~~ CharsWhileIn("0-9").rep(min = 1, max = 5).! ~~ Index).map { (i1, numStr: String, i2) =>
val num = numStr.toInt
if num > 0 && num < 65535 then
if num > 0 && num < 65535 then
numStr
else
error(at(i1,i2), s"Invalid port number: $numStr. Must be in range 0 <= port < 65536")
"0"
end if
end if
}
}

Expand Down
82 changes: 43 additions & 39 deletions .../shared/src/main/scala/com/ossuminc/riddl/language/parsing/ExtensibleTopLevelParser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ trait ExtensibleTopLevelParser(using PlatformContext)
SagaParser,
StreamingParser,
StatementParser,
TokenStreamParser,
ParsingContext {

def input: RiddlParserInput
Expand Down Expand Up @@ -87,30 +88,30 @@ trait ExtensibleTopLevelParser(using PlatformContext)
}
}

/** Obtain the parser for any of the main AST definition types */
/** Obtain the parser for any of the main AST definition types */
protected def parserFor[T <: Definition: ClassTag]: P[?] => P[T] = {
val parser: P[?] => P[?] = classTag[T].runtimeClass match {
case x if x == classOf[Adaptor] => adaptor(_)
case x if x == classOf[Author] => author(_)
case x if x == classOf[Connector] => connector(_)
case x if x == classOf[Constant] => constant(_)
case x if x == classOf[Adaptor] => adaptor(_)
case x if x == classOf[Author] => author(_)
case x if x == classOf[Connector] => connector(_)
case x if x == classOf[Constant] => constant(_)
case x if x == classOf[ContainedGroup] => containedGroup(_)
case x if x == classOf[Context] => context(_)
case x if x == classOf[Domain] => domain(_)
case x if x == classOf[Entity] => entity(_)
case x if x == classOf[Epic] => epic(_)
case x if x == classOf[Function] => function(_)
case x if x == classOf[Invariant] => invariant(_)
case x if x == classOf[Module] => module(_)
case x if x == classOf[Nebula] => nebula(_)
case x if x == classOf[Projector] => projector(_)
case x if x == classOf[Relationship] => relationship(_)
case x if x == classOf[Repository] => repository(_)
case x if x == classOf[Root] => root(_)
case x if x == classOf[Saga] => saga(_)
case x if x == classOf[Streamlet] => streamlet(_)
case x if x == classOf[Type] => typeDef(_)
case x if x == classOf[User] => user(_)
case x if x == classOf[Context] => context(_)
case x if x == classOf[Domain] => domain(_)
case x if x == classOf[Entity] => entity(_)
case x if x == classOf[Epic] => epic(_)
case x if x == classOf[Function] => function(_)
case x if x == classOf[Invariant] => invariant(_)
case x if x == classOf[Module] => module(_)
case x if x == classOf[Nebula] => nebula(_)
case x if x == classOf[Projector] => projector(_)
case x if x == classOf[Relationship] => relationship(_)
case x if x == classOf[Repository] => repository(_)
case x if x == classOf[Root] => root(_)
case x if x == classOf[Saga] => saga(_)
case x if x == classOf[Streamlet] => streamlet(_)
case x if x == classOf[Type] => typeDef(_)
case x if x == classOf[User] => user(_)
case _ =>
throw new RuntimeException(
s"No parser defined for ${classTag[T].runtimeClass}"
Expand All @@ -119,36 +120,32 @@ trait ExtensibleTopLevelParser(using PlatformContext)
parser.asInstanceOf[P[?] => P[T]]
}


/** Parse the input expecting the contents of a Root node
* @returns
* Either the failure error messages or the Root parsed
*/
* @returns
* Either the failure error messages or the Root parsed
*/
def parseRoot: Either[Messages, Root] = doParse[Root](root(_))

/** Parse the input expecting the contents of a Root node but also return the
* list of files that were read
* @returns
* Either the failure messages and a list of files or the Root that was parsed
* and the list of files parsed.
*/
/** Parse the input expecting the contents of a Root node but also return the list of files that were read
* @returns
* Either the failure messages and a list of files or the Root that was parsed and the list of files parsed.
*/
def parseRootWithURLs: Either[(Messages, Seq[URL]), (Root, Seq[URL])] = {
doParse[Root](root(_)) match {
case l @ Left(messages) => Left(messages -> this.getURLs)
case r @ Right(root) => Right(root -> this.getURLs)
}
}

/** Parse the input expecting main definitions in any order, a nebula. Each
* definition must be syntactically correct but the top level definitions do
* not require the hierarchical structure of parsing for Root contents.
* @returns
* Either the failure messages or the Nebula of definitions
*/
/** Parse the input expecting main definitions in any order, a nebula. Each definition must be syntactically correct
* but the top level definitions do not require the hierarchical structure of parsing for Root contents.
* @returns
* Either the failure messages or the Nebula of definitions
*/
def parseNebula: Either[Messages, Nebula] = doParse[Nebula](nebula(_))

/** Parse the input expecting definitions in any order, a nebula. Each definition must be syntactically correct
* but the top level definitions do not require the hierarchical structure of parsing for Root contents.
/** Parse the input expecting definitions in any order, a nebula. Each definition must be syntactically correct but
* the top level definitions do not require the hierarchical structure of parsing for Root contents.
* @returns
* Either the failure messages with the list of parsed URL or the Nebula of definitions with the list of parsed
* URLs
Expand All @@ -159,4 +156,11 @@ trait ExtensibleTopLevelParser(using PlatformContext)
case r @ Right(nebula) => Right(nebula -> this.getURLs)
}
}

def parseTokens: Either[Messages, List[Token]] = {
parse[List[Token]](input, parseAllTokens(_)) match
case Left((messages, _)) => Left(messages)
case Right((list, _)) => Right(list)
end match
}
}
Loading

0 comments on commit 2747020

Please sign in to comment.