chore: prepare 2.8.1 (#1006)

* fix: broken query in TS migration * refactor: TS testing infrastructure to work with given DS
SwissDataScienceCenter · Jun 1, 2022 · 15a90ad · 15a90ad
1 parent cfb6381
commit 15a90ad
Show file tree

Hide file tree

Showing 12 changed files with 69 additions and 68 deletions.
diff --git a/generators/src/main/scala/io/renku/generators/Generators.scala b/generators/src/main/scala/io/renku/generators/Generators.scala
@@ -180,24 +180,20 @@ object Generators {
 
   val httpPorts: Gen[Int Refined Positive] = choose(2000, 10000) map Refined.unsafeApply
 
-  def httpUrls(pathGenerator: Gen[String] = relativePaths(minSegments = 0, maxSegments = 2)): Gen[String] =
+  def httpUrls(hostGenerator: Gen[String] = nonEmptyStrings(),
+               pathGenerator: Gen[String] = relativePaths(minSegments = 0, maxSegments = 2)
+  ): Gen[String] =
     for {
-      protocol <- Arbitrary.arbBool.arbitrary map {
-                    case true  => "http"
-                    case false => "https"
-                  }
-      port <- httpPorts
-      host <- nonEmptyStrings()
-      path <- pathGenerator
+      protocol <- Gen.oneOf("http", "https")
+      port     <- httpPorts
+      host     <- hostGenerator
+      path     <- pathGenerator
       pathValidated = if (path.isEmpty) "" else s"/$path"
     } yield s"$protocol://$host:$port$pathValidated"
 
   val localHttpUrls: Gen[String] = for {
-    protocol <- Arbitrary.arbBool.arbitrary map {
-                  case true  => "http"
-                  case false => "https"
-                }
-    port <- httpPorts
+    protocol <- Gen.oneOf("http", "https")
+    port     <- httpPorts
   } yield s"$protocol://localhost:$port"
 
   lazy val semanticVersions: Gen[String] = Gen.listOfN(3, positiveInts(max = 150)).map(_.mkString("."))

diff --git a/graph-commons/src/test/scala/io/renku/rdfstore/InMemoryRdfStore.scala b/graph-commons/src/test/scala/io/renku/rdfstore/InMemoryRdfStore.scala
@@ -47,7 +47,6 @@ import java.net.{ServerSocket, SocketException}
 import java.nio.charset.StandardCharsets.UTF_8
 import scala.language.reflectiveCalls
 import scala.util.Random.nextInt
-import scala.xml.Elem
 
 trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with ResultsDecoder {
   this: Suite with IOSpec =>
@@ -88,13 +87,13 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result
     .fromString(s"$fusekiBaseUrl/${rdfStoreConfig.datasetName}/sparql")
     .fold(throw _, identity)
 
-  private lazy val rdfConnectionResource: Resource[IO, RDFConnection] =
-    Resource.make(openConnection)(connection => IO(connection.close()))
+  private def rdfConnectionResource(dsName: DatasetName): Resource[IO, RDFConnection] =
+    Resource.make(openConnection(dsName))(connection => IO(connection.close()))
 
-  private def openConnection: IO[RDFConnection] = IO {
+  private def openConnection(dsName: DatasetName): IO[RDFConnection] = IO {
     RDFConnectionFuseki
       .create()
-      .destination((fusekiBaseUrl / rdfStoreConfig.datasetName).toString)
+      .destination((fusekiBaseUrl / dsName).toString)
       .build()
   }
 
@@ -116,51 +115,49 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result
     super.afterAll()
   }
 
-  protected def loadToStore(triples: Elem): Unit = rdfConnectionResource
-    .use { connection =>
-      IO {
-        connection.load {
-          ModelFactory.createDefaultModel.read(new ByteArrayInputStream(triples.toString().getBytes), "")
-        }
-      }
-    }
-    .unsafeRunSync()
+  protected def loadToStore(jsonLD: JsonLD): Unit =
+    loadToStore(rdfStoreConfig.datasetName, "default", jsonLD)
 
-  protected def loadToStore(triples: JsonLD): Unit = rdfConnectionResource
-    .use { connection =>
-      IO {
-        connection.load {
-          val model = ModelFactory.createDefaultModel()
-          RDFDataMgr.read(model, new ByteArrayInputStream(triples.toJson.noSpaces.getBytes(UTF_8)), null, Lang.JSONLD)
-          model
-        }
-      }
-    }
-    .unsafeRunSync()
+  protected def loadToStore(jsonLDs: JsonLD*): Unit = {
+    val jsonLD = JsonLD.arr(jsonLDs.flatMap(_.flatten.toOption.flatMap(_.asArray).getOrElse(List.empty[JsonLD])): _*)
+    loadToStore(rdfStoreConfig.datasetName, "default", jsonLD)
+  }
+
+  protected def loadToStore(graphId: EntityId, jsonLD: JsonLD): Unit =
+    loadToStore(rdfStoreConfig.datasetName, graphId.show, jsonLD)
+
+  protected def loadToStore(graphId: EntityId, jsonLDs: JsonLD*): Unit = {
+    val jsonLD = JsonLD.arr(jsonLDs.flatMap(_.flatten.toOption.flatMap(_.asArray).getOrElse(List.empty[JsonLD])): _*)
+    loadToStore(rdfStoreConfig.datasetName, graphId.show, jsonLD)
+  }
+
+  protected def loadToStore(dsName: DatasetName, graphId: EntityId, jsonLD: JsonLD): Unit =
+    loadToStore(dsName, graphId.show, jsonLD)
 
-  protected def loadToStore(jsonLDs: JsonLD*): Unit = rdfConnectionResource
+  private def loadToStore(dsName: DatasetName, graphId: String, jsonLD: JsonLD): Unit = rdfConnectionResource(dsName)
     .use { connection =>
       IO {
-        connection.load {
-          val model = ModelFactory.createDefaultModel()
-
-          val flattenedJsonLDs: Seq[JsonLD] =
-            jsonLDs.flatMap(_.flatten.toOption.flatMap(_.asArray).getOrElse(List.empty[JsonLD]))
+        val model = {
+          val mod = ModelFactory.createDefaultModel()
           RDFDataMgr.read(
-            model,
-            new ByteArrayInputStream(Json.arr(flattenedJsonLDs.map(_.toJson): _*).noSpaces.getBytes(UTF_8)),
+            mod,
+            new ByteArrayInputStream(jsonLD.toJson.noSpaces.getBytes(UTF_8)),
             null,
             Lang.JSONLD
           )
-          model
+          mod
         }
+
+        connection.load(graphId.show, model)
       }
     }
     .unsafeRunSync()
 
-  protected def loadToStore[T](objects: T*)(implicit encoder: JsonLDEncoder[T]): Unit = loadToStore(
-    objects.map(encoder.apply): _*
-  )
+  protected def loadToStore[T](objects: T*)(implicit encoder: JsonLDEncoder[T]): Unit =
+    loadToStore(objects.map(encoder.apply): _*)
+
+  protected def loadToStore[T](graphId: EntityId, objects: T*)(implicit encoder: JsonLDEncoder[T]): Unit =
+    loadToStore(graphId, objects.map(encoder.apply): _*)
 
   protected def insertTriple(entityId: EntityId, p: String, o: String): Unit =
     queryRunner
@@ -191,6 +188,9 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result
     import io.circe.Decoder._
     import io.renku.graph.model.Schemas._
 
+    def runQuery(query: SparqlQuery): IO[List[Map[String, String]]] =
+      queryExpecting[List[Map[String, String]]](query)
+
     def runQuery(query: String): IO[List[Map[String, String]]] =
       queryExpecting[List[Map[String, String]]] {
         SparqlQuery.of(
@@ -257,6 +257,8 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result
         .map(maybeValue => maybeValue map (varName -> _))
   }
 
+  protected def runQuery(query: SparqlQuery): IO[List[Map[String, String]]] = queryRunner.runQuery(query)
+
   protected def runQuery(query: String): IO[List[Map[String, String]]] = queryRunner.runQuery(query)
 
   protected def runUpdate(query: SparqlQuery): IO[Unit] = queryRunner.runUpdate(query)

diff --git a/renku-model/src/test/scala/io/renku/graph/model/GraphModelGenerators.scala b/renku-model/src/test/scala/io/renku/graph/model/GraphModelGenerators.scala
@@ -33,7 +33,8 @@ import scala.util.Random
 
 object GraphModelGenerators {
 
-  implicit val renkuBaseUrls: Gen[RenkuBaseUrl] = httpUrls() map RenkuBaseUrl.apply
+  implicit val renkuBaseUrls: Gen[RenkuBaseUrl] =
+    httpUrls(hostGenerator = nonEmptyStrings().map(_.toLowerCase)) map RenkuBaseUrl.apply
 
   implicit val gitLabUrls: Gen[GitLabUrl] = for {
     url  <- httpUrls()

diff --git a/renku-model/src/test/scala/io/renku/graph/model/projectsSpec.scala b/renku-model/src/test/scala/io/renku/graph/model/projectsSpec.scala
@@ -154,7 +154,7 @@ class ProjectResourceIdSpec extends AnyWordSpec with ScalaCheckPropertyChecks wi
   "instantiation" should {
 
     "be successful for URLs ending with a project path" in {
-      forAll(httpUrls(pathGenerator)) { url =>
+      forAll(httpUrls(pathGenerator = pathGenerator)) { url =>
         ResourceId(url).value shouldBe url
       }
     }
@@ -167,7 +167,7 @@ class ProjectResourceIdSpec extends AnyWordSpec with ScalaCheckPropertyChecks wi
 
     "fail when ending with a /" in {
       an[IllegalArgumentException] shouldBe thrownBy {
-        ResourceId(httpUrls(pathGenerator).generateOne + "/")
+        ResourceId(httpUrls(pathGenerator = pathGenerator).generateOne + "/")
       }
     }
   }

diff --git a/...odel/src/test/scala/io/renku/graph/model/testentities/generators/ActivityGenerators.scala b/...odel/src/test/scala/io/renku/graph/model/testentities/generators/ActivityGenerators.scala
@@ -130,7 +130,7 @@ trait ActivityGenerators {
     def generateList(projectDateCreated: projects.DateCreated): List[Activity] =
       factory(projectDateCreated).generateList()
 
-    def many: List[ActivityGenFactory] = List.fill(positiveInts(5).generateOne)(factory)
+    def multiple: List[ActivityGenFactory] = List.fill(positiveInts(5).generateOne)(factory)
 
     def withDateBefore(max: InstantTinyType): Gen[Activity] =
       factory(projects.DateCreated(max.value))

diff --git a/...c/test/scala/io/renku/graph/model/testentities/generators/DatasetEntitiesGenerators.scala b/...c/test/scala/io/renku/graph/model/testentities/generators/DatasetEntitiesGenerators.scala
@@ -23,7 +23,7 @@ import cats.data.NonEmptyList
 import cats.syntax.all._
 import eu.timepit.refined.auto._
 import io.renku.generators.Generators.Implicits._
-import io.renku.generators.Generators.{fixed, nonBlankStrings, sentences, timestamps}
+import io.renku.generators.Generators.{fixed, nonBlankStrings, positiveInts, sentences, timestamps}
 import io.renku.graph.model.GraphModelGenerators.{datasetCreatedDates, datasetDescriptions, datasetExternalSameAs, datasetIdentifiers, datasetImageUris, datasetInternalSameAs, datasetKeywords, datasetLicenses, datasetNames, datasetOriginalIdentifiers, datasetPartExternals, datasetPartIds, datasetPartSources, datasetPublishedDates, datasetTitles, datasetVersions, projectCreatedDates}
 import io.renku.graph.model._
 import io.renku.graph.model.datasets.{DerivedFrom, ExternalSameAs, Identifier, InternalSameAs, OriginalIdentifier, TopmostSameAs}
@@ -244,6 +244,8 @@ trait DatasetEntitiesGenerators {
     def generateList(projectDateCreated: projects.DateCreated): List[Dataset[P]] =
       factory(projectDateCreated).generateList()
 
+    def multiple: List[DatasetGenFactory[P]] = List.fill(positiveInts(5).generateOne)(factory)
+
     def createMultiple(max: Int): List[DatasetGenFactory[P]] = List.fill(Random.nextInt(max - 1) + 1)(factory)
 
     def toGeneratorFor(project: RenkuProject): Gen[Dataset[P]] = factory(project.dateCreated)

diff --git a/...t/scala/io/renku/graph/model/testentities/generators/RenkuProjectEntitiesGenerators.scala b/...t/scala/io/renku/graph/model/testentities/generators/RenkuProjectEntitiesGenerators.scala
@@ -62,11 +62,11 @@ trait RenkuProjectEntitiesGenerators {
       )
 
   def renkuProjectEntities(
-      visibilityGen:       Gen[Visibility],
-      minDateCreated:      projects.DateCreated = projects.DateCreated(Instant.EPOCH),
-      activitiesFactories: List[ActivityGenFactory] = Nil,
-      datasetsFactories:   List[DatasetGenFactory[Dataset.Provenance]] = Nil,
-      forksCountGen:       Gen[ForksCount] = anyForksCount
+      visibilityGen:     Gen[Visibility],
+      minDateCreated:    projects.DateCreated = projects.DateCreated(Instant.EPOCH),
+      activityFactories: List[ActivityGenFactory] = Nil,
+      datasetFactories:  List[DatasetGenFactory[Dataset.Provenance]] = Nil,
+      forksCountGen:     Gen[ForksCount] = anyForksCount
   ): Gen[RenkuProject.WithoutParent] = for {
     path             <- projectPaths
     name             <- Gen.const(path.toName)
@@ -79,8 +79,8 @@ trait RenkuProjectEntitiesGenerators {
     keywords         <- projectKeywords.toGeneratorOfSet(minElements = 0)
     members          <- personEntities(withGitLabId).toGeneratorOfSet(minElements = 0)
     version          <- projectSchemaVersions
-    activities       <- activitiesFactories.map(_.apply(dateCreated)).sequence
-    datasets         <- datasetsFactories.map(_.apply(dateCreated)).sequence
+    activities       <- activityFactories.map(_.apply(dateCreated)).sequence
+    datasets         <- datasetFactories.map(_.apply(dateCreated)).sequence
   } yield RenkuProject.WithoutParent(path,
                                      name,
                                      maybeDescription,

diff --git a/...ator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnly.scala b/...ator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnly.scala
@@ -22,7 +22,7 @@ package migrations
 import cats.effect.Async
 import cats.syntax.all._
 import eu.timepit.refined.auto._
-import io.renku.graph.model.Schemas.{renku, schema}
+import io.renku.graph.model.Schemas.{prov, renku, schema}
 import io.renku.metrics.MetricsRegistry
 import io.renku.rdfstore.SparqlQuery.Prefixes
 import io.renku.rdfstore.{SparqlQuery, SparqlQueryTimeRecorder}
@@ -37,7 +37,7 @@ private object MultipleTopmostDerivedFromOnly {
   private lazy val name = Migration.Name("Multiple topmostDerivedFrom on Modified DS only")
   private[migrations] lazy val query = SparqlQuery.of(
     name.asRefined,
-    Prefixes of (schema -> "schema", renku -> "renku"),
+    Prefixes of (prov -> "prov", renku -> "renku", schema -> "schema"),
     s"""|SELECT DISTINCT ?path
         |WHERE {
         |  ?dsId a schema:Dataset;

diff --git a/...vents/categories/triplesgenerated/transformation/activities/ActivityTransformerSpec.scala b/...vents/categories/triplesgenerated/transformation/activities/ActivityTransformerSpec.scala
@@ -42,7 +42,7 @@ class ActivityTransformerSpec extends AnyWordSpec with should.Matchers with Mock
     "create update queries for changed/deleted activities' authors " +
       "and associations' agents" in new TestCase {
         val project = anyRenkuProjectEntities
-          .withActivities(activityEntities(planEntities()).modify(toAssociationPersonAgent).many: _*)
+          .withActivities(activityEntities(planEntities()).modify(toAssociationPersonAgent).multiple: _*)
           .generateOne
           .to[entities.RenkuProject]
 

diff --git a/...sgenerator/events/categories/tsmigrationrequest/migrations/MalformedActivityIdsSpec.scala b/...sgenerator/events/categories/tsmigrationrequest/migrations/MalformedActivityIdsSpec.scala
@@ -59,7 +59,7 @@ class MalformedActivityIdsSpec extends AnyWordSpec with should.Matchers with IOS
         anyRenkuProjectEntities.generateOne.to[entities.RenkuProject]
       )
 
-      runQuery(MalformedActivityIds.query.toString)
+      runQuery(MalformedActivityIds.query)
         .unsafeRunSync()
         .map(row => projects.Path(row("path")))
         .toSet shouldBe Set(project1.path, project2.path)

diff --git a/...esgenerator/events/categories/tsmigrationrequest/migrations/MalformedDSImageIdsSpec.scala b/...esgenerator/events/categories/tsmigrationrequest/migrations/MalformedDSImageIdsSpec.scala
@@ -67,7 +67,7 @@ class MalformedDSImageIdsSpec extends AnyWordSpec with should.Matchers with IOSp
         anyRenkuProjectEntities.generateOne.to[entities.RenkuProject]
       )
 
-      runQuery(MalformedDSImageIds.query.toString)
+      runQuery(MalformedDSImageIds.query)
         .unsafeRunSync()
         .map(row => projects.Path(row("path")))
         .toSet shouldBe Set(project1.path, project2.path)

diff --git a/.../events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnlySpec.scala b/.../events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnlySpec.scala
@@ -57,7 +57,7 @@ class MultipleTopmostDerivedFromOnlySpec extends AnyWordSpec with should.Matcher
       val illegalTopmost2 = datasetTopmostDerivedFroms.generateOne
       insertTriple(modifiedDSProject2.entityId, "renku:topmostDerivedFrom", illegalTopmost2.showAs[RdfResource])
 
-      runQuery(MultipleTopmostDerivedFromOnly.query.toString)
+      runQuery(MultipleTopmostDerivedFromOnly.query)
         .unsafeRunSync()
         .map(row => projects.Path(row("path")))
         .toSet shouldBe Set(brokenProject1.path, brokenProject2.path)