Skip to content

Commit

Permalink
Hotfix for #92 & #93 (#94)
Browse files Browse the repository at this point in the history 
* fix bug with  statement.fetchSize(...)
* use reator java-agent for clear stacktraces
* use (Mono|Flux).usingWhen(...) for executing SQL-requests on connections
* improve clone detector
* log thread name
* change postgres configuration
  • Loading branch information
@lamtev
lamtev authored Jun 13, 2020
1 parent 65cc6d6 commit 5456ed9
Show file tree
Hide file tree
Showing 25 changed files with 482 additions and 491 deletions.
2 changes: 1 addition & 1 deletion api/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ RUN gradle build -x test -x web:build --no-daemon

FROM openjdk:14-alpine
COPY --from=build /app/api/build/libs/*.jar accula.jar
ENTRYPOINT ["java","-jar","/accula.jar"]
ENTRYPOINT ["java","--enable-preview","-jar","/accula.jar"]
18 changes: 16 additions & 2 deletions api/build.gradle.kts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
plugins {
id("org.springframework.boot") version "2.3.0.RELEASE"
id("io.spring.dependency-management") version "1.0.9.RELEASE"
id("net.bytebuddy.byte-buddy-gradle-plugin") version "1.10.11"
}

version = "1.0-SNAPSHOT"

val byteBuddyPlugin: Configuration by configurations.creating

dependencies {
implementation("org.springframework.boot:spring-boot-starter-webflux")
testImplementation("org.springframework.boot:spring-boot-starter-test") {
Expand All @@ -13,7 +16,11 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-security")
testImplementation("org.springframework.security:spring-security-test")

testImplementation("io.projectreactor:reactor-test:3.3.6.RELEASE")
implementation("net.bytebuddy:byte-buddy:1.10.11")
implementation("io.projectreactor:reactor-tools")
byteBuddyPlugin(group = "io.projectreactor", name = "reactor-tools", classifier = "original")

testImplementation("io.projectreactor:reactor-test")

implementation("org.springframework.boot:spring-boot-starter-actuator")

Expand All @@ -25,7 +32,7 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-data-r2dbc")
implementation("io.r2dbc:r2dbc-postgresql:0.8.2.RELEASE")
implementation("io.r2dbc:r2dbc-pool:0.8.2.RELEASE")
implementation("io.r2dbc:r2dbc-spi:0.8.1.RELEASE")
implementation("io.r2dbc:r2dbc-spi:0.8.2.RELEASE")

implementation("org.slf4j:slf4j-api:2.0.0-alpha1")
implementation("org.slf4j:slf4j-log4j12:2.0.0-alpha1")
Expand All @@ -40,3 +47,10 @@ dependencies {

annotationProcessor("org.springframework.boot:spring-boot-configuration-processor")
}

byteBuddy {
transformation(closureOf<net.bytebuddy.build.gradle.Transformation> {
plugin = "reactor.tools.agent.ReactorDebugByteBuddyPlugin"
setClassPath(byteBuddyPlugin)
})
}
3 changes: 3 additions & 0 deletions api/src/main/java/org/accula/api/AcculaApiApplication.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import reactor.tools.agent.ReactorDebugAgent;

@SpringBootApplication
@SuppressWarnings("PMD.UseUtilityClass")
public class AcculaApiApplication {
public static void main(final String[] args) {
ReactorDebugAgent.init();
SpringApplication.run(AcculaApiApplication.class, args);
}
}
6 changes: 3 additions & 3 deletions api/src/main/java/org/accula/api/code/CodeLoaderImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,11 @@ public Flux<FileEntity> getFiles(final CommitSnapshot snapshot) {
@Override
public Flux<FileEntity> getFiles(final CommitSnapshot snapshot, final FileFilter filter) {
return getRepository(snapshot)
.publishOn(scheduler)
.switchIfEmpty(Mono.error(REPO_NOT_FOUND))
.publishOn(scheduler)
.flatMapMany(repo -> Flux.fromIterable(getObjectLoaders(repo, snapshot.getSha())))
.filter(filenameAndLoader -> filter.test(filenameAndLoader.getT1()))
.map(filenameAndLoader -> new FileEntity(snapshot, filenameAndLoader.getT1(), getFileContent(filenameAndLoader.getT2())))
.switchIfEmpty(Mono.error(CUT_ERROR))
.onErrorResume(e -> e instanceof MissingObjectException, e -> {
log.info("Most probably branch with the commit {} has been deleted: {}", snapshot.toString(), e);
return Flux.empty();
Expand All @@ -75,8 +74,8 @@ public Flux<FileEntity> getFiles(final CommitSnapshot snapshot, final FileFilter
@Override
public Mono<FileEntity> getFile(final CommitSnapshot snapshot, final String filename) {
return getRepository(snapshot)
.publishOn(scheduler)
.switchIfEmpty(Mono.error(REPO_NOT_FOUND))
.publishOn(scheduler)
.flatMap(repo -> Mono.justOrEmpty(getObjectLoader(repo, snapshot.getSha(), filename)))
.switchIfEmpty(Mono.error(FILE_NOT_FOUND))
.map(loader -> new FileEntity(snapshot, filename, getFileContent(loader)))
Expand All @@ -98,6 +97,7 @@ public Mono<FileEntity> getFileSnippet(final CommitSnapshot snapshot, final Stri
.switchIfEmpty(Mono.error(CUT_ERROR));
}

@Override
public Flux<Tuple2<FileEntity, FileEntity>> getDiff(final CommitSnapshot base, final CommitSnapshot head) {
return getDiff(base, head, FileFilter.ALL);
}
Expand Down
3 changes: 2 additions & 1 deletion api/src/main/java/org/accula/api/code/FileFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ public interface FileFilter extends Predicate<String> {
FileFilter TESTS = file -> file.contains("Test");
FileFilter SRC_JAVA = JAVA.and(TESTS.negate());

default FileFilter and(FileFilter other) {
@Override
default FileFilter and(Predicate<? super String> other) {
return f -> test(f) && other.test(f);
}

Expand Down
6 changes: 6 additions & 0 deletions api/src/main/java/org/accula/api/config/DbConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io.r2dbc.spi.ConnectionFactories;
import io.r2dbc.spi.ConnectionFactoryOptions;
import lombok.RequiredArgsConstructor;
import org.accula.api.db.repo.ConnectionProvidedRepo;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
Expand Down Expand Up @@ -54,4 +55,9 @@ public ConnectionPool connectionFactory() {

return new ConnectionPool(poolConfig);
}

@Bean
public ConnectionProvidedRepo.ConnectionProvider connectionProvider(final ConnectionPool connectionPool) {
return connectionPool::create;
}
}
10 changes: 5 additions & 5 deletions api/src/main/java/org/accula/api/config/WebConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
@RequiredArgsConstructor
@EnableConfigurationProperties(WebhookProperties.class)
public class WebConfig implements WebFluxConfigurer {
private final CurrentUserRepo currentUserRepository;
private final CurrentUserRepo currentUserRepo;

@Bean
public WebClient webClient() {
Expand All @@ -34,14 +34,14 @@ public WebClient webClient() {

@Bean
public GithubClient.AccessTokenProvider githubAccessTokenProvider() {
return () -> currentUserRepository
return () -> currentUserRepo
.get()
.flatMap(user -> Mono.justOrEmpty(user.getGithubAccessToken()));
.flatMap(user -> Mono.just(user.getGithubAccessToken()));
}

@Bean
public GithubClient.LoginProvider githubLoginProvider() {
return () -> currentUserRepository
return () -> currentUserRepo
.get()
.flatMap(user -> Mono.just(user.getGithubUser().getLogin()));
}
Expand All @@ -54,6 +54,6 @@ public RepositoryProvider repositoryManager(@Value("${accula.reposPath}") final

@Bean
public CloneDetector cloneDetector() {
return new PrimitiveCloneDetector(5, 7);
return new PrimitiveCloneDetector(3, 8);
}
}
144 changes: 71 additions & 73 deletions api/src/main/java/org/accula/api/db/repo/CloneRepoImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package org.accula.api.db.repo;

import io.r2dbc.pool.ConnectionPool;
import io.r2dbc.postgresql.api.PostgresqlStatement;
import io.r2dbc.spi.Connection;
import io.r2dbc.spi.Row;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.accula.api.db.model.Clone;
import org.springframework.stereotype.Component;
Expand All @@ -19,8 +19,9 @@
*/
@Component
@RequiredArgsConstructor
public final class CloneRepoImpl implements CloneRepo {
private final ConnectionPool connectionPool;
public final class CloneRepoImpl implements CloneRepo, ConnectionProvidedRepo {
@Getter
private final ConnectionProvider connectionProvider;

@Override
public Mono<Clone> insert(final Clone clone) {
Expand All @@ -32,19 +33,18 @@ public Flux<Clone> insert(final Collection<Clone> clones) {
if (clones.isEmpty()) {
return Flux.empty();
}

final var cloneList = clones instanceof ArrayList ? (ArrayList<Clone>) clones : new ArrayList<>(clones);

return connectionPool
.create()
.flatMapMany(connection -> {
final var statement = insertStatement(connection);
cloneList.forEach(clone -> applyInsertBindings(clone, statement).add());
statement.fetchSize(cloneList.size());
return manyWithConnection(connection -> {
final var statement = insertStatement(connection);
cloneList.forEach(clone -> applyInsertBindings(clone, statement).add());

return Repos.convertMany(statement.execute(), connection, row -> Converters
.value(row, "id", Long.class))
.zipWithIterable(cloneList, (id, clone) -> clone.toBuilder().id(id).build());
});
return statement
.execute()
.flatMap(result -> ConnectionProvidedRepo.column(result, "id", Long.class))
.zipWithIterable(cloneList, (id, clone) -> clone.toBuilder().id(id).build());
});
}

@Override
Expand All @@ -54,66 +54,64 @@ public Mono<Clone> findById(final Long id) {

@Override
public Flux<Clone> findByTargetCommitSnapshotSha(final String sha) {
return connectionPool
.create()
.flatMapMany(connection -> Mono
.from(connection
.createStatement("""
SELECT clone.id AS id,
target.sha AS target_sha,
target.branch AS target_branch,
target_snap_to_pull.pull_id AS target_pull_id,
target_repo.id AS target_repo_id,
target_repo.name AS target_repo_name,
target_repo.description AS target_repo_description,
target_repo_owner.id AS target_repo_owner_id,
target_repo_owner.login AS target_repo_owner_login,
target_repo_owner.name AS target_repo_owner_name,
target_repo_owner.avatar AS target_repo_owner_avatar,
target_repo_owner.is_org AS target_repo_owner_is_org,
clone.target_file AS target_file,
clone.target_from_line AS target_from_line,
clone.target_to_line AS target_to_line,
source.sha AS source_sha,
source.branch AS source_branch,
source_snap_to_pull.pull_id AS source_pull_id,
source_repo.id AS source_repo_id,
source_repo.name AS source_repo_name,
source_repo.description AS source_repo_description,
source_repo_owner.id AS source_repo_owner_id,
source_repo_owner.login AS source_repo_owner_login,
source_repo_owner.name AS source_repo_owner_name,
source_repo_owner.avatar AS source_repo_owner_avatar,
source_repo_owner.is_org AS source_repo_owner_is_org,
clone.source_file AS source_file,
clone.source_from_line AS source_from_line,
clone.source_to_line AS source_to_line
FROM clone
JOIN commit_snapshot target
ON clone.target_commit_sha = target.sha
AND clone.target_repo_id = target.repo_id
JOIN repo_github target_repo
ON target.repo_id = target_repo.id
JOIN user_github target_repo_owner
ON target_repo.owner_id = target_repo_owner.id
JOIN commit_snapshot_pull target_snap_to_pull
ON target.sha = target_snap_to_pull.commit_snapshot_sha
AND target.repo_id = target_snap_to_pull.commit_snapshot_repo_id
JOIN commit_snapshot source
ON clone.source_commit_sha = source.sha
AND clone.source_repo_id = source.repo_id
JOIN repo_github source_repo
ON source.repo_id = source_repo.id
JOIN user_github source_repo_owner
ON source_repo.owner_id = source_repo_owner.id
JOIN commit_snapshot_pull source_snap_to_pull
ON source.sha = source_snap_to_pull.commit_snapshot_sha
AND source.repo_id = source_snap_to_pull.commit_snapshot_repo_id
WHERE clone.target_commit_sha = $1
""")
.bind("$1", sha)
.execute())
.flatMapMany(result -> Repos.convertMany(result, connection, this::convert)));
return manyWithConnection(connection -> Mono
.from(connection
.createStatement("""
SELECT clone.id AS id,
target.sha AS target_sha,
target.branch AS target_branch,
target_snap_to_pull.pull_id AS target_pull_id,
target_repo.id AS target_repo_id,
target_repo.name AS target_repo_name,
target_repo.description AS target_repo_description,
target_repo_owner.id AS target_repo_owner_id,
target_repo_owner.login AS target_repo_owner_login,
target_repo_owner.name AS target_repo_owner_name,
target_repo_owner.avatar AS target_repo_owner_avatar,
target_repo_owner.is_org AS target_repo_owner_is_org,
clone.target_file AS target_file,
clone.target_from_line AS target_from_line,
clone.target_to_line AS target_to_line,
source.sha AS source_sha,
source.branch AS source_branch,
source_snap_to_pull.pull_id AS source_pull_id,
source_repo.id AS source_repo_id,
source_repo.name AS source_repo_name,
source_repo.description AS source_repo_description,
source_repo_owner.id AS source_repo_owner_id,
source_repo_owner.login AS source_repo_owner_login,
source_repo_owner.name AS source_repo_owner_name,
source_repo_owner.avatar AS source_repo_owner_avatar,
source_repo_owner.is_org AS source_repo_owner_is_org,
clone.source_file AS source_file,
clone.source_from_line AS source_from_line,
clone.source_to_line AS source_to_line
FROM clone
JOIN commit_snapshot target
ON clone.target_commit_sha = target.sha
AND clone.target_repo_id = target.repo_id
JOIN repo_github target_repo
ON target.repo_id = target_repo.id
JOIN user_github target_repo_owner
ON target_repo.owner_id = target_repo_owner.id
JOIN commit_snapshot_pull target_snap_to_pull
ON target.sha = target_snap_to_pull.commit_snapshot_sha
AND target.repo_id = target_snap_to_pull.commit_snapshot_repo_id
JOIN commit_snapshot source
ON clone.source_commit_sha = source.sha
AND clone.source_repo_id = source.repo_id
JOIN repo_github source_repo
ON source.repo_id = source_repo.id
JOIN user_github source_repo_owner
ON source_repo.owner_id = source_repo_owner.id
JOIN commit_snapshot_pull source_snap_to_pull
ON source.sha = source_snap_to_pull.commit_snapshot_sha
AND source.repo_id = source_snap_to_pull.commit_snapshot_repo_id
WHERE clone.target_commit_sha = $1
""")
.bind("$1", sha)
.execute())
.flatMapMany(result -> ConnectionProvidedRepo.convertMany(result, this::convert)));
}

private static PostgresqlStatement insertStatement(final Connection connection) {
Expand Down
Loading

0 comments on commit 5456ed9

Please sign in to comment.