Skip to content

Commit

Permalink
chore: prepare 2.9.0 (#1020)
Browse files Browse the repository at this point in the history 
* feat: added support for Orcid as Person id (#1015)

* chore: extended grace period for events in Generating Triples

* chore: CLI upgraded to 1.4.0

* chore: raising HTTP timeouts on connection to TS (#1009)

* fix: transformation flow to update changed Project properties (#1017)

* fix: TS migration fixing duplicate topmostDerivedFrom

* fix: DS with multiple descriptions (#1016)

* fix: multiple topmostSameAs issue on DS transformation flow (#1014)

* fix: CLEAN_UP events handling process not to die on failure (#1013)

* fix: TS_MIGRATION_REQUEST keeps a long-running transaction (#1011)

* refactor: improved readability of the renkuBaseUrl class (#1019)
  • Loading branch information
@jachro
jachro authored Jun 13, 2022
1 parent 15a90ad commit cc31a47
Show file tree
Hide file tree
Showing 158 changed files with 2,580 additions and 1,521 deletions.
2 changes: 1 addition & 1 deletion acceptance-tests/src/test/resources/application.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ services {
}
renku {
url = "https://dev.renku.ch"
resources-url = "http://localhost:9004/knowledge-graph"
api-url = "http://localhost:9004/knowledge-graph"
}
}

Expand Down
4 changes: 2 additions & 2 deletions acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/data/package.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ import java.time.Instant.now

package object data extends RdfStoreData {

implicit val cliVersion: CliVersion = currentVersionPair.cliVersion
val renkuResourcesUrl: renku.ResourcesUrl = renku.ResourcesUrl("http://localhost:9004/knowledge-graph")
implicit val cliVersion: CliVersion = currentVersionPair.cliVersion
val renkuApiUrl: renku.ApiUrl = renku.ApiUrl("http://localhost:9004/knowledge-graph")

def dataProjects(
projectGen: Gen[testentities.RenkuProject],
Expand Down
8 changes: 4 additions & 4 deletions .../src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/DatasetsResourcesSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,8 +472,8 @@ trait DatasetsResources {
}"""
.deepMerge(
_links(
Rel("details") -> Href(renkuResourcesUrl / "datasets" / dataset.identification.identifier),
Rel("initial-version") -> Href(renkuResourcesUrl / "datasets" / dataset.provenance.originalIdentifier)
Rel("details") -> Href(renkuApiUrl / "datasets" / dataset.identification.identifier),
Rel("initial-version") -> Href(renkuApiUrl / "datasets" / dataset.provenance.originalIdentifier)
)
)
.deepMerge(provenanceEncoder(dataset.provenance))
Expand Down Expand Up @@ -512,7 +512,7 @@ trait DatasetsResources {
.addIfDefined("description" -> dataset.additionalInfo.maybeDescription)
.deepMerge {
_links(
Rel("details") -> Href(renkuResourcesUrl / "datasets" / actualIdentifier)
Rel("details") -> Href(renkuApiUrl / "datasets" / actualIdentifier)
)
}
}
Expand All @@ -529,7 +529,7 @@ trait DatasetsResources {
}

private implicit lazy val personEncoder: Encoder[Person] = Encoder.instance[Person] {
case Person(name, maybeEmail, _, _) => json"""{
case Person(name, maybeEmail, _, _, _) => json"""{
"name": $name
}""" addIfDefined ("email" -> maybeEmail)
}
Expand Down
15 changes: 7 additions & 8 deletions acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/knowledgegraph/package.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,15 @@ package io.renku.graph.acceptancetests

package object knowledgegraph {

import io.circe.literal._
import io.circe.{Encoder, Json}
import io.renku.graph.acceptancetests.data.Project.Permissions._
import io.renku.graph.acceptancetests.data.Project.{Urls, _}
import io.renku.graph.acceptancetests.data.{Project, _}
import io.renku.graph.model.projects.{DateCreated, ForksCount}
import io.renku.graph.model.testentities
import io.renku.http.rest.Links.{Href, Link, Rel, _links}
import io.circe.literal._
import io.renku.graph.acceptancetests.data.{Project, _}
import io.renku.graph.model.testentities._
import eu.timepit.refined.auto._
import io.renku.http.rest.Links.{Href, Link, Rel, _links}

def fullJson(project: Project): Json = json"""{
"identifier": ${project.id.value},
Expand All @@ -53,8 +52,8 @@ package object knowledgegraph {
"version": ${project.entitiesProject.version.value}
}""" deepMerge {
_links(
Link(Rel.Self -> Href(renkuResourcesUrl / "projects" / project.path)),
Link(Rel("datasets") -> Href(renkuResourcesUrl / "projects" / project.path / "datasets"))
Link(Rel.Self -> Href(renkuApiUrl / "projects" / project.path)),
Link(Rel("datasets") -> Href(renkuApiUrl / "projects" / project.path / "datasets"))
)
} deepMerge {
project.entitiesProject.maybeDescription
Expand Down Expand Up @@ -98,11 +97,11 @@ package object knowledgegraph {
}

private implicit lazy val personEncoder: Encoder[Person] = Encoder.instance {
case Person(name, Some(email), _, _) => json"""{
case Person(name, Some(email), _, _, _) => json"""{
"name": ${name.value},
"email": ${email.value}
}"""
case Person(name, _, _, _) => json"""{
case Person(name, _, _, _, _) => json"""{
"name": ${name.value}
}"""
}
Expand Down
2 changes: 1 addition & 1 deletion ...ce-tests/src/test/scala/io/renku/graph/acceptancetests/stubs/RemoteTriplesGenerator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ trait RemoteTriplesGenerator {
def `GET <triples-generator>/projects/:id/commits/:id returning OK with some triples`(
project: data.Project,
commitId: CommitId
)(implicit gitLabApiUrl: GitLabApiUrl, renkuBaseUrl: RenkuBaseUrl): Unit =
)(implicit gitLabApiUrl: GitLabApiUrl, renkuUrl: RenkuUrl): Unit =
`GET <triples-generator>/projects/:id/commits/:id returning OK`(
project,
commitId,
Expand Down
6 changes: 3 additions & 3 deletions acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/tooling/GraphServices.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ import io.renku.graph.acceptancetests.db.{EventLog, TokenRepository}
import io.renku.graph.acceptancetests.stubs.{GitLab, RemoteTriplesGenerator}
import io.renku.graph.acceptancetests.tooling.KnowledgeGraphClient.KnowledgeGraphClient
import io.renku.graph.acceptancetests.tooling.WebhookServiceClient.WebhookServiceClient
import io.renku.graph.config.RenkuBaseUrlLoader
import io.renku.graph.model.RenkuBaseUrl
import io.renku.graph.config.RenkuUrlLoader
import io.renku.graph.model.RenkuUrl
import io.renku.rdfstore.FusekiBaseUrl
import io.renku.testtools.IOSpec
import org.scalatest.{BeforeAndAfterAll, Suite}
Expand All @@ -38,7 +38,7 @@ trait GraphServices extends GitLab with RemoteTriplesGenerator with IOSpec with
self: Suite =>

protected implicit val fusekiBaseUrl: FusekiBaseUrl = RDFStore.fusekiBaseUrl
implicit val renkuBaseUrl: RenkuBaseUrl = RenkuBaseUrlLoader[Try]().fold(throw _, identity)
implicit val renkuUrl: RenkuUrl = RenkuUrlLoader[Try]().fold(throw _, identity)
implicit lazy val logger: Logger[IO] = TestLogger()

val restClient: RestClientImpl = new RestClientImpl()
Expand Down
4 changes: 2 additions & 2 deletions acceptance-tests/src/test/scala/io/renku/graph/acceptancetests/tooling/RDFStore.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import io.renku.graph.acceptancetests.data.RdfStoreData
import RdfStoreData.currentVersionPair
import io.renku.graph.model.RenkuVersionPair
import io.renku.graph.model.Schemas._
import io.renku.graph.model.testentities.renkuBaseUrl
import io.renku.graph.model.testentities.renkuUrl
import io.renku.jsonld.EntityId
import io.renku.rdfstore.FusekiBaseUrl
import org.apache.jena.fuseki.main.FusekiServer
Expand Down Expand Up @@ -107,7 +107,7 @@ object RDFStore {
_ = maybeVersionPair.map { currentVersionPair =>
newJena.connection.update(s"""
INSERT DATA{
<${EntityId.of((renkuBaseUrl / "version-pair").toString)}> a <${renku / "VersionPair"}> ;
<${EntityId.of((renkuUrl / "version-pair").toString)}> a <${renku / "VersionPair"}> ;
<${renku / "cliVersion"}> '${currentVersionPair.cliVersion}' ;
<${renku / "schemaVersion"}> '${currentVersionPair.schemaVersion}'}""")
}
Expand Down
6 changes: 3 additions & 3 deletions event-log/src/main/scala/io/renku/eventlog/events/categories/creation/EventPersister.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@ private class EventPersisterImpl[F[_]: MonadCancelThrow: SessionResource](
Kleisli { case (transaction, session) =>
for {
sp <- transaction.savepoint
result <- insertIfNotDuplicate(event)(session) recoverWith { case error =>
transaction.rollback(sp) >> error.raiseError[F, Result]
}
result <- insertIfNotDuplicate(event)(session)
.flatTap(_ => transaction.commit)
.recoverWith { case error => transaction.rollback(sp) >> error.raiseError[F, Result] }
_ <- whenA(aNewEventIsCreated(result))(waitingEventsGauge.increment(event.project.path))
} yield result
}
Expand Down
8 changes: 6 additions & 2 deletions ...t-log/src/main/scala/io/renku/eventlog/events/categories/statuschange/StatusChanger.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,12 @@ private class StatusChangerImpl[F[_]: MonadCancelThrow: SessionResource](gaugesU
Kleisli { case (transaction, session) =>
{
for {
savepoint <- Kleisli.liftF(transaction.savepoint)
updateResults <- dbUpdater.updateDB(event) recoverWith rollback(transaction)(savepoint)(event)
savepoint <- Kleisli.liftF(transaction.savepoint)
updateResults <-
dbUpdater
.updateDB(event)
.flatMapF(res => transaction.commit.map(_ => res))
.recoverWith(rollback(transaction)(savepoint)(event))
_ <- Kleisli.liftF(updateGauges(updateResults)) recoverWith { case NonFatal(_) => Kleisli.pure(()) }
} yield ()
} run session
Expand Down
10 changes: 5 additions & 5 deletions ...t-log/src/main/scala/io/renku/eventlog/subscriptions/tsmigrationrequest/EventFinder.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ private class EventFinder[F[_]: Async: SessionResource](queriesExecTimes: Labele
savepoint <- transaction.savepoint
maybeEventCandidate <- tryToPop(session)
maybeEvent <- verifySingleEventTaken(maybeEventCandidate)(session) >>= {
case true => maybeEventCandidate.pure[F]
case true => transaction.commit >> maybeEventCandidate.pure[F]
case false => (transaction rollback savepoint).map(_ => Option.empty[MigrationRequestEvent])
}
} yield maybeEvent
}
}

private def tryToPop = findEvent.flatMap {
private def tryToPop = findEvent >>= {
case Some(event) => markTaken(event) map toNoneIfTaken(event)
case _ => Kleisli.pure(Option.empty[MigrationRequestEvent])
}
Expand All @@ -79,7 +79,7 @@ private class EventFinder[F[_]: Async: SessionResource](queriesExecTimes: Labele
ORDER BY change_date DESC
LIMIT 1
) latest
JOIN ts_migration m on m.subscriber_version = latest.subscriber_version
JOIN ts_migration m ON m.subscriber_version = latest.subscriber_version
ORDER BY m.change_date DESC
""".query(subscriberUrlDecoder ~ serviceVersionDecoder ~ migrationStatusDecoder ~ changeDateDecoder)
.map { case url ~ version ~ status ~ changeDate => (url, version, status, changeDate) }
Expand Down Expand Up @@ -154,8 +154,8 @@ private class EventFinder[F[_]: Async: SessionResource](queriesExecTimes: Labele
"""
.query(int8)
.map {
case 0 | 1 => true
case _ => false
case 0L | 1L => true
case _ => false
}
)
.arguments(event.subscriberVersion)
Expand Down
2 changes: 1 addition & 1 deletion ...c/main/scala/io/renku/eventlog/subscriptions/zombieevents/LongProcessingEventFinder.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ private class LongProcessingEventFinder[F[_]: Async: SessionResource](
}

private lazy val findGracePeriod: ProcessingStatus => EventProcessingTime = {
case GeneratingTriples => EventProcessingTime(Duration ofDays 4 * 7)
case GeneratingTriples => EventProcessingTime(Duration ofDays 6 * 7)
case TransformingTriples => EventProcessingTime(Duration ofDays 1)
case Deleting => EventProcessingTime(Duration ofDays 1)
}
Expand Down
10 changes: 5 additions & 5 deletions ...t/scala/io/renku/eventlog/events/categories/migrationstatuschange/StatusUpdaterSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class StatusUpdaterSpec

updater.updateStatus(event).unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe event.newStatus -> ChangeDate(now)
findRow(url, version) shouldBe event.newStatus -> ChangeDate(now)

if (Set(RecoverableFailure, NonRecoverableFailure) contains event.newStatus)
findMessage(url, version) shouldBe message.some
Expand All @@ -76,7 +76,7 @@ class StatusUpdaterSpec

updater.updateStatus(ToDone(url, version)).unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe status -> changeDate
findRow(url, version) shouldBe status -> changeDate
}
}

Expand All @@ -95,9 +95,9 @@ class StatusUpdaterSpec

updater.updateStatus(ToDone(url, version)).unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe Done -> ChangeDate(now)
findRows(otherUrl, version) shouldBe Sent -> changeDate
findRows(url, otherVersion) shouldBe Sent -> changeDate
findRow(url, version) shouldBe Done -> ChangeDate(now)
findRow(otherUrl, version) shouldBe Sent -> changeDate
findRow(url, otherVersion) shouldBe Sent -> changeDate
}
}

Expand Down
14 changes: 7 additions & 7 deletions .../test/scala/io/renku/eventlog/subscriptions/tsmigrationrequest/DispatchRecoverySpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class DispatchRecoverySpec
.returnToQueue(MigrationRequestEvent(url, version), reason = notBusyStatus.generateOne)
.unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe New -> ChangeDate(now)
findRow(url, version) shouldBe New -> ChangeDate(now)
}

"leave the status of the corresponding row in the ts_migration table as Sent " +
Expand All @@ -73,7 +73,7 @@ class DispatchRecoverySpec
.returnToQueue(MigrationRequestEvent(url, version), reason = TemporarilyUnavailable)
.unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe Sent -> ChangeDate(now)
findRow(url, version) shouldBe Sent -> ChangeDate(now)
}

MigrationStatus.all - Sent foreach { status =>
Expand All @@ -84,7 +84,7 @@ class DispatchRecoverySpec
.returnToQueue(MigrationRequestEvent(url, version), reason = sendingResults.generateOne)
.unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe status -> changeDate
findRow(url, version) shouldBe status -> changeDate
}
}

Expand All @@ -98,7 +98,7 @@ class DispatchRecoverySpec
)
.unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe status -> changeDate
findRow(url, version) shouldBe status -> changeDate
}
}

Expand All @@ -111,7 +111,7 @@ class DispatchRecoverySpec

recovery.recover(url, MigrationRequestEvent(url, version))(exception).unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe NonRecoverableFailure -> ChangeDate(now)
findRow(url, version) shouldBe NonRecoverableFailure -> ChangeDate(now)
findMessage(url, version) shouldBe MigrationMessage(exception).some

logger.loggedOnly(Info("TS_MIGRATION_REQUEST: recovering from NonRecoverable Failure", exception))
Expand All @@ -123,7 +123,7 @@ class DispatchRecoverySpec

recovery.recover(url, MigrationRequestEvent(url, version))(exception).unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe status -> changeDate
findRow(url, version) shouldBe status -> changeDate
}
}

Expand All @@ -135,7 +135,7 @@ class DispatchRecoverySpec

recovery.recover(failingEvent.subscriberUrl, failingEvent)(exception).unsafeRunSync() shouldBe ()

findRows(url, version) shouldBe status -> changeDate
findRow(url, version) shouldBe status -> changeDate
}
}

Expand Down
Loading

0 comments on commit cc31a47

Please sign in to comment.