TokenParser Enhancements

* Add a "literal code" statement to everything_full.riddl
* Enhance the token parser to accept that syntax
* Adjust test cases affected by the test input change
* Add "LiteralCode" token case.

language/input/everything_full.riddl

@@ -94,7 +94,15 @@ context full is {
     function whenUnderTheInfluence is {
       requires { n: Nothing }
       returns  { b: Boolean }
-        "aribtrary statement"
+      "aribtrary statement"
+      ```scala
+        // Simulate a creative state
+        val randomFactor = Math.random() // A random value between 0 and 1
+        val threshold = 0.7 // Threshold for creativity
+
+        // If the random factor exceeds the threshold, consider it a creative state
+        b = randomFactor > threshold
+      ```
     } with {
       briefly as "Something is nothing interesting"
     }

language/jvm/src/test/scala/com/ossuminc/riddl/language/JVMASTTest.scala

@@ -25,7 +25,7 @@ class JVMASTTest extends AbstractTestingBasis {
       ud.format must be(url.toExternalForm)
       val lines: scala.collection.Seq[String] = ud.lines.map(_.s)
       val head = lines.head
-      head must include("resolver")
+      head must include("ossuminc")
     }
   }
 }

language/shared/src/main/scala/com/ossuminc/riddl/language/AST.scala

@@ -3926,7 +3926,7 @@ object AST:
     case Predefined(at: At) extends Token(at)
     case Keyword(at: At) extends Token(at)
     case Comment(at: At) extends Token(at)
-    case LiteralString(at: At) extends Token(at)
+    case LiteralCode(at: At) extends Token(at)
     case MarkdownLine(at: At) extends Token(at)
     case Identifier(at: At) extends Token(at)
     case Numeric(at: At) extends Token(at)

language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/CommonParser.scala

@@ -71,7 +71,7 @@ private[parsing] trait CommonParser(using io: PlatformContext)
   }
 
   def literalStrings[u: P]: P[Seq[LiteralString]] = { P(literalString.rep(1)) }
-
+  
   def markdownLines[u: P]: P[Seq[LiteralString]] = {
     P(markdownLine.rep(1))
   }

language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/Keywords.scala

@@ -195,7 +195,7 @@ object Keywords {
   def nebula[u: P]: P[Unit] = keyword(Keyword.nebula)
 
   def on[u: P]: P[Unit] = keyword(Keyword.on)
-
+  
   def onInit[u: P]: P[Unit] = keyword("on init")
 
   def onOther[u: P]: P[Unit] = keyword("on other")

language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/NoWhiteSpaceParsers.scala

@@ -97,11 +97,12 @@ private[parsing] trait NoWhiteSpaceParsers {
 
   private final val escape_chars = "\\\\\\\"aefnrt"
   private def shortcut[u: P]: P[String] = P("\\" ~ CharIn(escape_chars)).!
+
   def escape[u: P]: P[String] = P(shortcut | hexEscape | unicodeEscape).!./
 
-  private def stringChars(c: Char): Boolean = c != '\"' && c != '\\'
+  def stringChars(c: Char): Boolean = c != '\"' && c != '\\'
 
-  private def strChars[u: P]: P[String] = P(CharsWhile(stringChars)).!./
+  def strChars[u: P]: P[String] = P(CharsWhile(stringChars)).!./
 
   def literalString[u: P]: P[LiteralString] = {
     P(

language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/Punctuation.scala

@@ -10,6 +10,7 @@ import fastparse.*
 object Punctuation {
   final val asterisk = "*"
   final val atSign = "@"
+  final val codeQuote = "```"
   final val comma = ","
   final val colon = ":"
   final val curlyOpen = "{"
@@ -48,6 +49,31 @@ object Punctuation {
   )
 
   def anyPunctuation[u: P]: P[Unit] = {
+    P(
+      StringIn(
+        asterisk,
+        atSign,
+        codeQuote,
+        comma,
+        colon,
+        curlyOpen,
+        curlyClose,
+        dot,
+        equalsSign,
+        plus,
+        question,
+        quote,
+        roundOpen,
+        roundClose,
+        squareOpen,
+        squareClose,
+        verticalBar,
+        undefinedMark
+      )
+    )
+  }
+
+  def tokenPunctuation[u: P]: P[Unit] = {
     P(
       StringIn(
         asterisk,

language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/TokenParser.scala

@@ -22,29 +22,53 @@ import scala.util.control.NonFatal
 
 trait TokenParser extends CommonParser with Readability {
 
-  private def numericToken[u:P]: P[Token.Numeric] = {
+  private def numericToken[u: P]: P[Token.Numeric] = {
     P(Index ~~ integer ~~ Index)./.map { case (start, _, end) => Token.Numeric(at(start, end)) }
   }
 
   private def punctuationToken[u: P]: P[Token.Punctuation] = {
-    P(Index ~~ Punctuation.anyPunctuation ~~ Index)./.map { case (start, end) => Token.Punctuation(at(start, end)) }
+    P(Index ~~ Punctuation.tokenPunctuation ~~ Index)./.map { case (start, end) =>
+      Token.Punctuation(at(start, end))
+    }
+  }
+
+  private def notCodeQuote[u:P]: P[Unit] = {
+    P(AnyChar.rep(1))
   }
+
+  private def literalCode[u: P]: P[Token.LiteralCode] = {
+    P(
+      Index ~~ Punctuation.codeQuote ~~ until3('`','`','`') ~~ Index
+    ).map { case (start: Int, _: String, end: Int) =>
+      Token.LiteralCode(at(start, end))
+    }
+  }
+
+  private def stringContent[u: P]: P[Unit] = P(CharsWhile(stringChars) | escape)
 
   private def quotedStringToken[u: P]: P[Token.QuotedString] = {
-    P(literalString)./.map { case litStr: LiteralString => Token.QuotedString(litStr.loc) }
+    P(
+      Index ~~ Punctuation.quote ~~/ stringContent.rep ~~ Punctuation.quote ~~ Index
+    )./.map { case (start: Int, end: Int) => Token.QuotedString(at(start, end)) }
   }
 
   private def readabilityToken[u: P]: P[Token.Readability] = {
-    P(Index ~~ anyReadability ~~ Index)./.map { case (start, end) => Token.Readability(at(start, end)) }
+    P(Index ~~ anyReadability ~~ Index)./.map { case (start, end) =>
+      Token.Readability(at(start, end))
+    }
   }
 
   private def predefinedToken[u: P]: P[Token.Predefined] = {
     import com.ossuminc.riddl.language.parsing.PredefType.*
-    P(Index ~~ PredefTypes.anyPredefType ~~ Index)./.map { case (start, end) => Token.Predefined(at(start, end)) }
+    P(Index ~~ Keywords.keywords(PredefTypes.anyPredefType) ~~ Index)./.map { case (start, end) =>
+      Token.Predefined(at(start, end))
+    }
   }
 
   private def keywordToken[u: P]: P[Token.Keyword] = {
-    P(Index ~~ Keywords.anyKeyword ~~ Index)./.map { case (start, end) => Token.Keyword(at(start, end)) }
+    P(Index ~~ Keywords.anyKeyword ~~ Index)./.map { case (start, end) =>
+      Token.Keyword(at(start, end))
+    }
   }
 
   private def commentToken[u: P]: P[Token.Comment] = {
@@ -62,17 +86,20 @@ trait TokenParser extends CommonParser with Readability {
   }
 
   private def otherToken[u: P]: P[Token.Other] = {
-    P(Index ~~ AnyChar.rep(1) ~~ Index)./.map { case (start, end) =>
+    P(
+      Index ~~ (!(CharIn(" \n\r") | End) ~~ AnyChar).rep(1) ~~ Index
+    )./.map { case (start, end) =>
       Token.Other(at(start, end))
     }
   }
 
   def parseAnyToken[u: P]: P[Token] = {
     P(
-      keywordToken |
-        punctuationToken |
-        quotedStringToken |
+      quotedStringToken |
         markdownLinesToken |
+        literalCode |
+        punctuationToken |
+        keywordToken |
         readabilityToken |
         predefinedToken |
         identifierToken |
@@ -83,6 +110,6 @@ trait TokenParser extends CommonParser with Readability {
   }
 
   def parseAllTokens[u: P]: P[List[Token]] = {
-    P(Start ~ parseAnyToken.rep(0) ~ End).map(_.toList)
+    P(Start ~ parseAnyToken.rep(1) ~ End).map(_.toList)
   }
 }

language/shared/src/main/scala/com/ossuminc/riddl/language/parsing/TypeParser.scala

@@ -318,8 +318,9 @@ private[parsing] trait TypeParser {
 
   private def uniqueIdType[u: P]: P[UniqueId] = {
     (Index ~ PredefType.Id ~ Punctuation.roundOpen ~/
-      maybe(Keyword.entity) ~ pathIdentifier ~ Punctuation.roundClose ~/ Index) map { case (start, pid, end) =>
-      UniqueId(at(start, end), pid)
+      maybe(Keyword.entity) ~ pathIdentifier ~ Punctuation.roundClose ~/ Index) map {
+      case (start, pid, end) =>
+        UniqueId(at(start, end), pid)
     }
   }
 
@@ -328,13 +329,16 @@ private[parsing] trait TypeParser {
   }
 
   def enumerator[u: P]: P[Enumerator] = {
-    P(Index ~~ identifier ~ enumValue  ~ withMetaData ~~Index).map { case (start, id, value, metaData, end) =>
-      Enumerator(at(start, end), id, value, metaData.toContents)
+    P(Index ~~ identifier ~ enumValue ~ withMetaData ~~ Index).map {
+      case (start, id, value, metaData, end) =>
+        Enumerator(at(start, end), id, value, metaData.toContents)
     }
   }
 
   private def enumerators[u: P]: P[Seq[Enumerator]] = {
-    enumerator.rep(1, maybe(Punctuation.comma)) | undefined[u, Seq[Enumerator]](Seq.empty[Enumerator])
+    enumerator.rep(1, maybe(Punctuation.comma)) | undefined[u, Seq[Enumerator]](
+      Seq.empty[Enumerator]
+    )
 
   }
 
@@ -348,7 +352,9 @@ private[parsing] trait TypeParser {
     P(
       Index ~ Keywords.one ~ of.? ~/ open ~
         (Punctuation.undefinedMark.!.map(_ => Seq.empty[AliasedTypeExpression]) |
-          aliasedTypeExpression.rep(0, P("or" | "|" | ","))) ~ close ~/ Index
+          aliasedTypeExpression
+            .rep(0, P(Keywords.or | Punctuation.verticalBar | Punctuation.comma))) ~ close
+        ~/ Index
     ).map { case (start, contents, end) =>
       Alternation(at(start, end), contents.toContents)
     }
@@ -525,16 +531,18 @@ private[parsing] trait TypeParser {
           Punctuation.plus
         ).!.? ~/ Index
     ).map {
-      case (start, None, None, typ, Some("?"), end)                     => Optional(at(start, end), typ)
-      case (start, None, None, typ, Some("+"), end)                     => OneOrMore(at(start, end), typ)
-      case (start, None, None, typ, Some("*"), end)                     => ZeroOrMore(at(start, end), typ)
-      case (start, Some("many"), None, typ, None, end)                  => OneOrMore(at(start, end), typ)
-      case (start, None, Some("optional"), typ, None, end)              => Optional(at(start, end), typ)
-      case (start, Some("many"), Some("optional"), typ, None, end)      => ZeroOrMore(at(start, end), typ)
-      case (start, None, Some("optional"), typ, Some("?"), end)         => Optional(at(start, end), typ)
-      case (start, Some("many"), None, typ, Some("+"), end)             => OneOrMore(at(start, end), typ)
-      case (start, Some("many"), Some("optional"), typ, Some("*"), end) => ZeroOrMore(at(start, end), typ)
-      case (_, None, None, typ, None, _)                                => typ
+      case (start, None, None, typ, Some("?"), end)        => Optional(at(start, end), typ)
+      case (start, None, None, typ, Some("+"), end)        => OneOrMore(at(start, end), typ)
+      case (start, None, None, typ, Some("*"), end)        => ZeroOrMore(at(start, end), typ)
+      case (start, Some("many"), None, typ, None, end)     => OneOrMore(at(start, end), typ)
+      case (start, None, Some("optional"), typ, None, end) => Optional(at(start, end), typ)
+      case (start, Some("many"), Some("optional"), typ, None, end) =>
+        ZeroOrMore(at(start, end), typ)
+      case (start, None, Some("optional"), typ, Some("?"), end) => Optional(at(start, end), typ)
+      case (start, Some("many"), None, typ, Some("+"), end)     => OneOrMore(at(start, end), typ)
+      case (start, Some("many"), Some("optional"), typ, Some("*"), end) =>
+        ZeroOrMore(at(start, end), typ)
+      case (_, None, None, typ, None, _) => typ
       case (start, _, _, typ, _, end) =>
         error(at(start, end), s"Cannot determine cardinality for $typ")
         typ