From 15a90adbf34c112d11cb6750b5a1527070ed7b8e Mon Sep 17 00:00:00 2001 From: Jakub Chrobasik Date: Wed, 1 Jun 2022 14:31:29 +0200 Subject: [PATCH] chore: prepare 2.8.1 (#1006) * fix: broken query in TS migration * refactor: TS testing infrastructure to work with given DS --- .../io/renku/generators/Generators.scala | 22 +++--- .../io/renku/rdfstore/InMemoryRdfStore.scala | 76 ++++++++++--------- .../graph/model/GraphModelGenerators.scala | 3 +- .../io/renku/graph/model/projectsSpec.scala | 4 +- .../generators/ActivityGenerators.scala | 2 +- .../DatasetEntitiesGenerators.scala | 4 +- .../RenkuProjectEntitiesGenerators.scala | 14 ++-- .../MultipleTopmostDerivedFromOnly.scala | 4 +- .../activities/ActivityTransformerSpec.scala | 2 +- .../migrations/MalformedActivityIdsSpec.scala | 2 +- .../migrations/MalformedDSImageIdsSpec.scala | 2 +- .../MultipleTopmostDerivedFromOnlySpec.scala | 2 +- 12 files changed, 69 insertions(+), 68 deletions(-) diff --git a/generators/src/main/scala/io/renku/generators/Generators.scala b/generators/src/main/scala/io/renku/generators/Generators.scala index 788a83b569..4a84a73cdf 100644 --- a/generators/src/main/scala/io/renku/generators/Generators.scala +++ b/generators/src/main/scala/io/renku/generators/Generators.scala @@ -180,24 +180,20 @@ object Generators { val httpPorts: Gen[Int Refined Positive] = choose(2000, 10000) map Refined.unsafeApply - def httpUrls(pathGenerator: Gen[String] = relativePaths(minSegments = 0, maxSegments = 2)): Gen[String] = + def httpUrls(hostGenerator: Gen[String] = nonEmptyStrings(), + pathGenerator: Gen[String] = relativePaths(minSegments = 0, maxSegments = 2) + ): Gen[String] = for { - protocol <- Arbitrary.arbBool.arbitrary map { - case true => "http" - case false => "https" - } - port <- httpPorts - host <- nonEmptyStrings() - path <- pathGenerator + protocol <- Gen.oneOf("http", "https") + port <- httpPorts + host <- hostGenerator + path <- pathGenerator pathValidated = if (path.isEmpty) "" else s"/$path" } yield s"$protocol://$host:$port$pathValidated" val localHttpUrls: Gen[String] = for { - protocol <- Arbitrary.arbBool.arbitrary map { - case true => "http" - case false => "https" - } - port <- httpPorts + protocol <- Gen.oneOf("http", "https") + port <- httpPorts } yield s"$protocol://localhost:$port" lazy val semanticVersions: Gen[String] = Gen.listOfN(3, positiveInts(max = 150)).map(_.mkString(".")) diff --git a/graph-commons/src/test/scala/io/renku/rdfstore/InMemoryRdfStore.scala b/graph-commons/src/test/scala/io/renku/rdfstore/InMemoryRdfStore.scala index e4fad6497c..249ea2b5fc 100644 --- a/graph-commons/src/test/scala/io/renku/rdfstore/InMemoryRdfStore.scala +++ b/graph-commons/src/test/scala/io/renku/rdfstore/InMemoryRdfStore.scala @@ -47,7 +47,6 @@ import java.net.{ServerSocket, SocketException} import java.nio.charset.StandardCharsets.UTF_8 import scala.language.reflectiveCalls import scala.util.Random.nextInt -import scala.xml.Elem trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with ResultsDecoder { this: Suite with IOSpec => @@ -88,13 +87,13 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result .fromString(s"$fusekiBaseUrl/${rdfStoreConfig.datasetName}/sparql") .fold(throw _, identity) - private lazy val rdfConnectionResource: Resource[IO, RDFConnection] = - Resource.make(openConnection)(connection => IO(connection.close())) + private def rdfConnectionResource(dsName: DatasetName): Resource[IO, RDFConnection] = + Resource.make(openConnection(dsName))(connection => IO(connection.close())) - private def openConnection: IO[RDFConnection] = IO { + private def openConnection(dsName: DatasetName): IO[RDFConnection] = IO { RDFConnectionFuseki .create() - .destination((fusekiBaseUrl / rdfStoreConfig.datasetName).toString) + .destination((fusekiBaseUrl / dsName).toString) .build() } @@ -116,51 +115,49 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result super.afterAll() } - protected def loadToStore(triples: Elem): Unit = rdfConnectionResource - .use { connection => - IO { - connection.load { - ModelFactory.createDefaultModel.read(new ByteArrayInputStream(triples.toString().getBytes), "") - } - } - } - .unsafeRunSync() + protected def loadToStore(jsonLD: JsonLD): Unit = + loadToStore(rdfStoreConfig.datasetName, "default", jsonLD) - protected def loadToStore(triples: JsonLD): Unit = rdfConnectionResource - .use { connection => - IO { - connection.load { - val model = ModelFactory.createDefaultModel() - RDFDataMgr.read(model, new ByteArrayInputStream(triples.toJson.noSpaces.getBytes(UTF_8)), null, Lang.JSONLD) - model - } - } - } - .unsafeRunSync() + protected def loadToStore(jsonLDs: JsonLD*): Unit = { + val jsonLD = JsonLD.arr(jsonLDs.flatMap(_.flatten.toOption.flatMap(_.asArray).getOrElse(List.empty[JsonLD])): _*) + loadToStore(rdfStoreConfig.datasetName, "default", jsonLD) + } + + protected def loadToStore(graphId: EntityId, jsonLD: JsonLD): Unit = + loadToStore(rdfStoreConfig.datasetName, graphId.show, jsonLD) + + protected def loadToStore(graphId: EntityId, jsonLDs: JsonLD*): Unit = { + val jsonLD = JsonLD.arr(jsonLDs.flatMap(_.flatten.toOption.flatMap(_.asArray).getOrElse(List.empty[JsonLD])): _*) + loadToStore(rdfStoreConfig.datasetName, graphId.show, jsonLD) + } + + protected def loadToStore(dsName: DatasetName, graphId: EntityId, jsonLD: JsonLD): Unit = + loadToStore(dsName, graphId.show, jsonLD) - protected def loadToStore(jsonLDs: JsonLD*): Unit = rdfConnectionResource + private def loadToStore(dsName: DatasetName, graphId: String, jsonLD: JsonLD): Unit = rdfConnectionResource(dsName) .use { connection => IO { - connection.load { - val model = ModelFactory.createDefaultModel() - - val flattenedJsonLDs: Seq[JsonLD] = - jsonLDs.flatMap(_.flatten.toOption.flatMap(_.asArray).getOrElse(List.empty[JsonLD])) + val model = { + val mod = ModelFactory.createDefaultModel() RDFDataMgr.read( - model, - new ByteArrayInputStream(Json.arr(flattenedJsonLDs.map(_.toJson): _*).noSpaces.getBytes(UTF_8)), + mod, + new ByteArrayInputStream(jsonLD.toJson.noSpaces.getBytes(UTF_8)), null, Lang.JSONLD ) - model + mod } + + connection.load(graphId.show, model) } } .unsafeRunSync() - protected def loadToStore[T](objects: T*)(implicit encoder: JsonLDEncoder[T]): Unit = loadToStore( - objects.map(encoder.apply): _* - ) + protected def loadToStore[T](objects: T*)(implicit encoder: JsonLDEncoder[T]): Unit = + loadToStore(objects.map(encoder.apply): _*) + + protected def loadToStore[T](graphId: EntityId, objects: T*)(implicit encoder: JsonLDEncoder[T]): Unit = + loadToStore(graphId, objects.map(encoder.apply): _*) protected def insertTriple(entityId: EntityId, p: String, o: String): Unit = queryRunner @@ -191,6 +188,9 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result import io.circe.Decoder._ import io.renku.graph.model.Schemas._ + def runQuery(query: SparqlQuery): IO[List[Map[String, String]]] = + queryExpecting[List[Map[String, String]]](query) + def runQuery(query: String): IO[List[Map[String, String]]] = queryExpecting[List[Map[String, String]]] { SparqlQuery.of( @@ -257,6 +257,8 @@ trait InMemoryRdfStore extends BeforeAndAfterAll with BeforeAndAfter with Result .map(maybeValue => maybeValue map (varName -> _)) } + protected def runQuery(query: SparqlQuery): IO[List[Map[String, String]]] = queryRunner.runQuery(query) + protected def runQuery(query: String): IO[List[Map[String, String]]] = queryRunner.runQuery(query) protected def runUpdate(query: SparqlQuery): IO[Unit] = queryRunner.runUpdate(query) diff --git a/renku-model/src/test/scala/io/renku/graph/model/GraphModelGenerators.scala b/renku-model/src/test/scala/io/renku/graph/model/GraphModelGenerators.scala index 5ae491368c..651d53ef2c 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/GraphModelGenerators.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/GraphModelGenerators.scala @@ -33,7 +33,8 @@ import scala.util.Random object GraphModelGenerators { - implicit val renkuBaseUrls: Gen[RenkuBaseUrl] = httpUrls() map RenkuBaseUrl.apply + implicit val renkuBaseUrls: Gen[RenkuBaseUrl] = + httpUrls(hostGenerator = nonEmptyStrings().map(_.toLowerCase)) map RenkuBaseUrl.apply implicit val gitLabUrls: Gen[GitLabUrl] = for { url <- httpUrls() diff --git a/renku-model/src/test/scala/io/renku/graph/model/projectsSpec.scala b/renku-model/src/test/scala/io/renku/graph/model/projectsSpec.scala index 22ef72491f..7a70c7c39c 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/projectsSpec.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/projectsSpec.scala @@ -154,7 +154,7 @@ class ProjectResourceIdSpec extends AnyWordSpec with ScalaCheckPropertyChecks wi "instantiation" should { "be successful for URLs ending with a project path" in { - forAll(httpUrls(pathGenerator)) { url => + forAll(httpUrls(pathGenerator = pathGenerator)) { url => ResourceId(url).value shouldBe url } } @@ -167,7 +167,7 @@ class ProjectResourceIdSpec extends AnyWordSpec with ScalaCheckPropertyChecks wi "fail when ending with a /" in { an[IllegalArgumentException] shouldBe thrownBy { - ResourceId(httpUrls(pathGenerator).generateOne + "/") + ResourceId(httpUrls(pathGenerator = pathGenerator).generateOne + "/") } } } diff --git a/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/ActivityGenerators.scala b/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/ActivityGenerators.scala index 0faab5de0c..5a5ec5f7c6 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/ActivityGenerators.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/ActivityGenerators.scala @@ -130,7 +130,7 @@ trait ActivityGenerators { def generateList(projectDateCreated: projects.DateCreated): List[Activity] = factory(projectDateCreated).generateList() - def many: List[ActivityGenFactory] = List.fill(positiveInts(5).generateOne)(factory) + def multiple: List[ActivityGenFactory] = List.fill(positiveInts(5).generateOne)(factory) def withDateBefore(max: InstantTinyType): Gen[Activity] = factory(projects.DateCreated(max.value)) diff --git a/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/DatasetEntitiesGenerators.scala b/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/DatasetEntitiesGenerators.scala index d884d69616..2d9a35f172 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/DatasetEntitiesGenerators.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/DatasetEntitiesGenerators.scala @@ -23,7 +23,7 @@ import cats.data.NonEmptyList import cats.syntax.all._ import eu.timepit.refined.auto._ import io.renku.generators.Generators.Implicits._ -import io.renku.generators.Generators.{fixed, nonBlankStrings, sentences, timestamps} +import io.renku.generators.Generators.{fixed, nonBlankStrings, positiveInts, sentences, timestamps} import io.renku.graph.model.GraphModelGenerators.{datasetCreatedDates, datasetDescriptions, datasetExternalSameAs, datasetIdentifiers, datasetImageUris, datasetInternalSameAs, datasetKeywords, datasetLicenses, datasetNames, datasetOriginalIdentifiers, datasetPartExternals, datasetPartIds, datasetPartSources, datasetPublishedDates, datasetTitles, datasetVersions, projectCreatedDates} import io.renku.graph.model._ import io.renku.graph.model.datasets.{DerivedFrom, ExternalSameAs, Identifier, InternalSameAs, OriginalIdentifier, TopmostSameAs} @@ -244,6 +244,8 @@ trait DatasetEntitiesGenerators { def generateList(projectDateCreated: projects.DateCreated): List[Dataset[P]] = factory(projectDateCreated).generateList() + def multiple: List[DatasetGenFactory[P]] = List.fill(positiveInts(5).generateOne)(factory) + def createMultiple(max: Int): List[DatasetGenFactory[P]] = List.fill(Random.nextInt(max - 1) + 1)(factory) def toGeneratorFor(project: RenkuProject): Gen[Dataset[P]] = factory(project.dateCreated) diff --git a/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/RenkuProjectEntitiesGenerators.scala b/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/RenkuProjectEntitiesGenerators.scala index c3f7d72ce8..e4515ab955 100644 --- a/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/RenkuProjectEntitiesGenerators.scala +++ b/renku-model/src/test/scala/io/renku/graph/model/testentities/generators/RenkuProjectEntitiesGenerators.scala @@ -62,11 +62,11 @@ trait RenkuProjectEntitiesGenerators { ) def renkuProjectEntities( - visibilityGen: Gen[Visibility], - minDateCreated: projects.DateCreated = projects.DateCreated(Instant.EPOCH), - activitiesFactories: List[ActivityGenFactory] = Nil, - datasetsFactories: List[DatasetGenFactory[Dataset.Provenance]] = Nil, - forksCountGen: Gen[ForksCount] = anyForksCount + visibilityGen: Gen[Visibility], + minDateCreated: projects.DateCreated = projects.DateCreated(Instant.EPOCH), + activityFactories: List[ActivityGenFactory] = Nil, + datasetFactories: List[DatasetGenFactory[Dataset.Provenance]] = Nil, + forksCountGen: Gen[ForksCount] = anyForksCount ): Gen[RenkuProject.WithoutParent] = for { path <- projectPaths name <- Gen.const(path.toName) @@ -79,8 +79,8 @@ trait RenkuProjectEntitiesGenerators { keywords <- projectKeywords.toGeneratorOfSet(minElements = 0) members <- personEntities(withGitLabId).toGeneratorOfSet(minElements = 0) version <- projectSchemaVersions - activities <- activitiesFactories.map(_.apply(dateCreated)).sequence - datasets <- datasetsFactories.map(_.apply(dateCreated)).sequence + activities <- activityFactories.map(_.apply(dateCreated)).sequence + datasets <- datasetFactories.map(_.apply(dateCreated)).sequence } yield RenkuProject.WithoutParent(path, name, maybeDescription, diff --git a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnly.scala b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnly.scala index cc9f49bed3..5eacd0db75 100644 --- a/triples-generator/src/main/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnly.scala +++ b/triples-generator/src/main/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnly.scala @@ -22,7 +22,7 @@ package migrations import cats.effect.Async import cats.syntax.all._ import eu.timepit.refined.auto._ -import io.renku.graph.model.Schemas.{renku, schema} +import io.renku.graph.model.Schemas.{prov, renku, schema} import io.renku.metrics.MetricsRegistry import io.renku.rdfstore.SparqlQuery.Prefixes import io.renku.rdfstore.{SparqlQuery, SparqlQueryTimeRecorder} @@ -37,7 +37,7 @@ private object MultipleTopmostDerivedFromOnly { private lazy val name = Migration.Name("Multiple topmostDerivedFrom on Modified DS only") private[migrations] lazy val query = SparqlQuery.of( name.asRefined, - Prefixes of (schema -> "schema", renku -> "renku"), + Prefixes of (prov -> "prov", renku -> "renku", schema -> "schema"), s"""|SELECT DISTINCT ?path |WHERE { | ?dsId a schema:Dataset; diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/triplesgenerated/transformation/activities/ActivityTransformerSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/triplesgenerated/transformation/activities/ActivityTransformerSpec.scala index 816916f23b..1421a90dfb 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/triplesgenerated/transformation/activities/ActivityTransformerSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/triplesgenerated/transformation/activities/ActivityTransformerSpec.scala @@ -42,7 +42,7 @@ class ActivityTransformerSpec extends AnyWordSpec with should.Matchers with Mock "create update queries for changed/deleted activities' authors " + "and associations' agents" in new TestCase { val project = anyRenkuProjectEntities - .withActivities(activityEntities(planEntities()).modify(toAssociationPersonAgent).many: _*) + .withActivities(activityEntities(planEntities()).modify(toAssociationPersonAgent).multiple: _*) .generateOne .to[entities.RenkuProject] diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedActivityIdsSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedActivityIdsSpec.scala index 406ea7f1f7..f912e6fcd0 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedActivityIdsSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedActivityIdsSpec.scala @@ -59,7 +59,7 @@ class MalformedActivityIdsSpec extends AnyWordSpec with should.Matchers with IOS anyRenkuProjectEntities.generateOne.to[entities.RenkuProject] ) - runQuery(MalformedActivityIds.query.toString) + runQuery(MalformedActivityIds.query) .unsafeRunSync() .map(row => projects.Path(row("path"))) .toSet shouldBe Set(project1.path, project2.path) diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedDSImageIdsSpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedDSImageIdsSpec.scala index 8f0c408365..4eafa08736 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedDSImageIdsSpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MalformedDSImageIdsSpec.scala @@ -67,7 +67,7 @@ class MalformedDSImageIdsSpec extends AnyWordSpec with should.Matchers with IOSp anyRenkuProjectEntities.generateOne.to[entities.RenkuProject] ) - runQuery(MalformedDSImageIds.query.toString) + runQuery(MalformedDSImageIds.query) .unsafeRunSync() .map(row => projects.Path(row("path"))) .toSet shouldBe Set(project1.path, project2.path) diff --git a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnlySpec.scala b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnlySpec.scala index f6ca2e7f13..5f9c830799 100644 --- a/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnlySpec.scala +++ b/triples-generator/src/test/scala/io/renku/triplesgenerator/events/categories/tsmigrationrequest/migrations/MultipleTopmostDerivedFromOnlySpec.scala @@ -57,7 +57,7 @@ class MultipleTopmostDerivedFromOnlySpec extends AnyWordSpec with should.Matcher val illegalTopmost2 = datasetTopmostDerivedFroms.generateOne insertTriple(modifiedDSProject2.entityId, "renku:topmostDerivedFrom", illegalTopmost2.showAs[RdfResource]) - runQuery(MultipleTopmostDerivedFromOnly.query.toString) + runQuery(MultipleTopmostDerivedFromOnly.query) .unsafeRunSync() .map(row => projects.Path(row("path"))) .toSet shouldBe Set(brokenProject1.path, brokenProject2.path)