Skip to content

Commit

Permalink
Merge pull request #187 from AlphaX-Projects/master
Browse files Browse the repository at this point in the history 
Update QDN file management
  • Loading branch information
@crowetic
crowetic authored Mar 25, 2024
2 parents f6b91df + df39819 commit cc740cc
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 44 deletions.
61 changes: 25 additions & 36 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,17 @@
<altcoinj.version>7dc8c6f</altcoinj.version>
<bitcoinj.version>0.15.10</bitcoinj.version>
<bouncycastle.version>1.69</bouncycastle.version>
<build-helper-maven-plugin.version>3.5.0</build-helper-maven-plugin.version>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
<ciyam-at.version>1.4.1</ciyam-at.version>
<commons-net.version>3.8.0</commons-net.version>
<commons-text.version>1.11.0</commons-text.version>
<commons-io.version>2.11.0</commons-io.version>
<commons-compress.version>1.25.0</commons-compress.version>
<commons-io.version>2.15.1</commons-io.version>
<commons-compress.version>1.26.1</commons-compress.version>
<commons-lang3.version>3.14.0</commons-lang3.version>
<dagger.version>1.2.2</dagger.version>
<extendedset.version>0.12.3</extendedset.version>
<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
<grpc.version>1.61.1</grpc.version>
<grpc.version>1.62.2</grpc.version>
<guava.version>33.0.0-jre</guava.version>
<hamcrest-library.version>2.2</hamcrest-library.version>
<homoglyph.version>1.2.1</homoglyph.version>
Expand All @@ -33,28 +32,30 @@
<jersey.version>2.41</jersey.version>
<jetty.version>9.4.54.v20240208</jetty.version>
<json-simple.version>1.1.1</json-simple.version>
<json.version>20240205</json.version>
<json.version>20240303</json.version>
<jsoup.version>1.17.2</jsoup.version>
<junit-jupiter-engine.version>5.10.0</junit-jupiter-engine.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<log4j.version>2.22.1</log4j.version>
<log4j.version>2.23.0</log4j.version>
<mail.version>1.5.0-b01</mail.version>
<maven-build-helper-plugin.version>3.5.0</maven-build-helper-plugin.version>
<maven-compiler-plugin.version>3.12.1</maven-compiler-plugin.version>
<maven-dependency-plugin.version>3.6.1</maven-dependency-plugin.version>
<maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
<maven-package-info-plugin.version>1.1.0</maven-package-info-plugin.version>
<maven-plugin.version>2.16.2</maven-plugin.version>
<maven-reproducible-build-plugin.version>0.16</maven-reproducible-build-plugin.version>
<maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
<maven-shade-plugin.version>3.5.1</maven-shade-plugin.version>
<maven-shade-plugin.version>3.5.2</maven-shade-plugin.version>
<maven-surefire-plugin.version>3.2.5</maven-surefire-plugin.version>
<package-info-maven-plugin.version>1.1.0</package-info-maven-plugin.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<protobuf.version>3.25.2</protobuf.version>
<protobuf.version>3.25.3</protobuf.version>
<replacer.version>1.5.3</replacer.version>
<reproducible-build-maven-plugin.version>0.16</reproducible-build-maven-plugin.version>
<simplemagic.version>1.17</simplemagic.version>
<slf4j.version>1.7.36</slf4j.version>
<swagger-api.version>2.0.10</swagger-api.version>
<swagger-ui.version>5.10.3</swagger-ui.version>
<swagger-ui.version>5.11.8</swagger-ui.version>
<upnp.version>1.2</upnp.version>
<versions-maven-plugin.version>2.16.2</versions-maven-plugin.version>
<xz.version>1.9</xz.version>
</properties>
<build>
Expand All @@ -70,7 +71,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>${versions-maven-plugin.version}</version>
<version>${maven-plugin.version}</version>
<configuration>
<generateBackupPoms>false</generateBackupPoms>
</configuration>
Expand Down Expand Up @@ -238,7 +239,7 @@
<plugin>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<version>${maven-package-info-plugin.version}</version>
<configuration>
<packages>
<package>
Expand Down Expand Up @@ -268,7 +269,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
<version>${maven-build-helper-plugin.version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
Expand Down Expand Up @@ -353,7 +354,7 @@
<plugin>
<groupId>io.github.zlika</groupId>
<artifactId>reproducible-build-maven-plugin</artifactId>
<version>${reproducible-build-maven-plugin.version}</version>
<version>${maven-reproducible-build-plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
Expand Down Expand Up @@ -388,15 +389,9 @@
<pluginExecutions>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
org.apache.maven.plugins
</groupId>
<artifactId>
maven-dependency-plugin
</artifactId>
<versionRange>
[3.6.0,)
</versionRange>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
<goals>
<goal>unpack</goal>
</goals>
Expand All @@ -407,15 +402,9 @@
</pluginExecution>
<pluginExecution>
<pluginExecutionFilter>
<groupId>
com.google.code.maven-replacer-plugin
</groupId>
<artifactId>
replacer
</artifactId>
<versionRange>
[1.5.3,)
</versionRange>
<groupId>com.google.code.maven-replacer-plugin</groupId>
<artifactId>replacer</artifactId>
<version>${replacer.version}</version>
<goals>
<goal>replace</goal>
</goals>
Expand Down Expand Up @@ -448,15 +437,15 @@
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
<version>${maven-build-helper-plugin.version}</version>
<scope>provided</scope>
<!-- needed for build, not for runtime -->
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.bohnman/package-info-maven-plugin -->
<dependency>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<version>${maven-package-info-plugin.version}</version>
<scope>provided</scope>
<!-- needed for build, not for runtime -->
</dependency>
Expand Down
21 changes: 20 additions & 1 deletion src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ public enum ResourceIdType {
public static int SHORT_DIGEST_LENGTH = 8;

protected Path filePath;
protected byte[] fileContent;
private boolean useTemporaryFile;

protected String hash58;
protected byte[] signature;
private ArrayList<ArbitraryDataFileChunk> chunks;
Expand Down Expand Up @@ -90,8 +93,14 @@ public ArbitraryDataFile(byte[] fileContent, byte[] signature, boolean useTempor
this.signature = signature;
LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));

this.fileContent = fileContent;
this.useTemporaryFile = useTemporaryFile;
}

public void save() throws DataException {

Path outputFilePath;
if (useTemporaryFile) {
if (this.useTemporaryFile) {
try {
outputFilePath = Files.createTempFile("qortalRawData", null);
outputFilePath.toFile().deleteOnExit();
Expand Down Expand Up @@ -149,6 +158,7 @@ public static ArbitraryDataFile fromTransactionData(ArbitraryTransactionData tra

case RAW_DATA:
arbitraryDataFile = ArbitraryDataFile.fromRawData(data, signature);
arbitraryDataFile.save();
break;
}

Expand Down Expand Up @@ -324,6 +334,7 @@ public int split(int chunkSize) throws DataException {
out.flush();

ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature);
chunk.save();
ValidationResult validationResult = chunk.isValid();
if (validationResult == ValidationResult.OK) {
this.chunks.add(chunk);
Expand Down Expand Up @@ -406,6 +417,10 @@ public boolean delete() {
}

public boolean delete(int attempts) {
if (this.filePath == null) {
return false;
}

// Keep trying to delete the data until it is deleted, or we reach 10 attempts
for (int i=0; i<attempts; i++) {
if (this.delete()) {
Expand Down Expand Up @@ -467,6 +482,10 @@ protected void cleanupFilesystem() throws IOException {
}

public byte[] getBytes() {
if (this.fileContent != null) {
return this.fileContent;
}

try {
return Files.readAllBytes(this.filePath);
} catch (IOException e) {
Expand Down
9 changes: 5 additions & 4 deletions src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,13 +242,14 @@ private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer,
boolean isRelayRequest = (requestingPeer != null);
if (isRelayRequest) {
if (!fileAlreadyExists) {
// File didn't exist locally before the request, and it's a forwarding request, so delete it
LOGGER.debug("Deleting file {} because it was needed for forwarding only", Base58.encode(hash));

// Keep trying to delete the data until it is deleted, or we reach 10 attempts
// File didn't exist locally before the request, and it's a forwarding request, so delete it if it exists.
// It shouldn't exist on the filesystem yet, but leaving this here just in case.
arbitraryDataFile.delete(10);
}
}
else {
arbitraryDataFile.save();
}

// If this is a metadata file then we need to update the cache
if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
Expand Down
12 changes: 9 additions & 3 deletions src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,11 +334,17 @@ public void onNetworkArbitraryMetadataMessage(Peer peer, Message message) {
}
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;

// Check if the name is blocked
boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));

// Save if not blocked
ArbitraryDataFile arbitraryMetadataFile = arbitraryMetadataMessage.getArbitraryMetadataFile();
if (!isBlocked && arbitraryMetadataFile != null) {
arbitraryMetadataFile.save();
}

// Forwarding
if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {

// Check if the name is blocked
boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
if (!isBlocked) {
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public void beforeTest() throws DataException {
public void testSplitAndJoin() throws DataException {
String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null, false);
arbitraryDataFile.save();
assertTrue(arbitraryDataFile.exists());
assertEquals(62, arbitraryDataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
Expand Down Expand Up @@ -51,6 +52,7 @@ public void testSplitAndJoinWithLargeFiles() throws DataException {
new Random().nextBytes(randomData); // No need for SecureRandom here

ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null, false);
arbitraryDataFile.save();
assertTrue(arbitraryDataFile.exists());
assertEquals(fileSize, arbitraryDataFile.size());
String originalFileDigest = arbitraryDataFile.digest58();
Expand Down

0 comments on commit cc740cc

Please sign in to comment.