Update the Scalariform library for tokenizing Scala submissions

The new version of Scalariform is 0.2.10. It is the last release
from 2019. The library is no longer maintained.

The built package scalariform.jar now includes the Scalariform library
as well as the small CLI script ScalariformTokens.
The JAR package is built for Scala 2.13 and can not be run with older
Scala versions.

Add sbt config files for building the scalariform.jar package with sbt.
The command `sbt assembly` builds the JAR package with the dependencies.
The build output is written to the scalariform/target/scala-2.13
directory and it must be copied from there manually.

If there is a need to move on from the dead Scalariform library,
one alternative could be Scalafmt:
https://scalameta.org/scalafmt/

.gitignore

@@ -19,6 +19,10 @@ Makefile
 !CMakeLists.txt
 compile_commands.json
 
+tokenizer/scalariform/target/
+tokenizer/scalariform/project/target/
+tokenizer/scalariform/project/project/target/
+
 **/bin
 *.a
 *.swp

tokenizer/scala.py

@@ -25,7 +25,7 @@ def tokenize(source, config):
 
     """
     try:
-        parsed = run(("scala", "-cp", "tokenizer/scalariform:tokenizer/scalariform/scalariform.jar", "ScalariformTokens"), source)
+        parsed = run(("scala", "-cp", "tokenizer/scalariform/scalariform.jar", "ScalariformTokens"), source)
         lines = parsed.decode("utf-8").split("\n", 1)
         return lines[0].strip(), index_string_to_list(lines[1].strip())
     except Exception as e:

tokenizer/scalariform/ScalariformTokens.scala

@@ -109,12 +109,12 @@ object ScalariformTokens extends App {
     Tokens.XML_UNPARSED -> '$',
     Tokens.XML_PROCESSING_INSTRUCTION -> '$')
 
-  val source = io.Source.stdin.getLines.mkString("\n")
-  val tokens = ScalaLexer.tokenise(source, scalaVersion = ScalaVersions.DEFAULT_VERSION)
+  val source = io.Source.stdin.getLines().mkString("\n")
+  val tokens = ScalaLexer.tokenise(source, scalaVersion = ScalaVersions.Scala_2_11.toString)
   val filtered = tokens.filter(t => !t.tokenType.isComment && !t.tokenType.isNewline && t.tokenType != Tokens.EOF)
 
   val ids = filtered.map(t => typeMap.getOrElse(t.tokenType, '?'))
-  val chars = filtered.map(t => t.offset + "-" + t.lastCharacterOffset)
+  val chars = filtered.map(t => s"${t.offset}-${t.lastCharacterOffset}")
 
   println(ids.mkString(""))
   println(chars.mkString(","))

tokenizer/scalariform/build.sbt

@@ -0,0 +1,15 @@
+ThisBuild / version := "0.1.0-SNAPSHOT"
+ThisBuild / scalaVersion := "2.13.8"
+
+scalacOptions ++= Seq("-deprecation", "-feature")
+
+// Scalariform is the library for parsing Scala code (the student's submissions).
+// The Scalariform project was stopped in 2019 and is no longer maintained.
+// https://mvnrepository.com/artifact/org.scalariform/scalariform
+libraryDependencies += "org.scalariform" %% "scalariform" % "0.2.10"
+
+// sbt-assembly plugin for building a JAR package that contains the dependencies as well.
+// But don't include the Scala standard library.
+assembly / mainClass := Some("ScalariformTokens")
+assemblyPackageScala / assembleArtifact := false
+

tokenizer/scalariform/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.6.2

tokenizer/scalariform/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0")