Skip to content

Commit

Permalink
Merge pull request #87 from CulverLab/bug-fixes
Browse files Browse the repository at this point in the history 
Bug fixes and enhancements
  • Loading branch information
@Chris-Schnaufer
Chris-Schnaufer authored Feb 23, 2023
2 parents 699b1a4 + 2f006e9 commit cd58b9b
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 27 deletions.
23 changes: 10 additions & 13 deletions Sanimal FX/src/main/java/model/image/Camtrap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import java.io.StringReader;
import java.security.InvalidParameterException;
import java.util.List;
import java.util.Vector;
import java.util.ArrayList;
import java.io.FileNotFoundException;
import java.io.IOException;
import com.opencsv.CSVReader;
Expand Down Expand Up @@ -68,7 +68,7 @@ public static final Camtrap instance(String folderPath) throws FileNotFoundExcep
CSVReader reader = new CSVReader(new FileReader(inFile));
String[] nextLine;

deployments = new Vector<Deployments>();
deployments = new ArrayList<Deployments>(Math.toIntExact(reader.getLinesRead()));
while ((nextLine = reader.readNext()) != null)
{
deployments.add(Deployments.instance(nextLine));
Expand All @@ -87,7 +87,7 @@ public static final Camtrap instance(String folderPath) throws FileNotFoundExcep
CSVReader reader = new CSVReader(new FileReader(inFile));
String[] nextLine;

media = new Vector<Media>();
media = new ArrayList<Media>(Math.toIntExact(reader.getLinesRead()));
while ((nextLine = reader.readNext()) != null)
{
media.add(Media.instance(nextLine));
Expand All @@ -106,7 +106,7 @@ public static final Camtrap instance(String folderPath) throws FileNotFoundExcep
CSVReader reader = new CSVReader(new FileReader(inFile));
String[] nextLine;

observations = new Vector<Observations>();
observations = new ArrayList<Observations>(Math.toIntExact(reader.getLinesRead()));
while ((nextLine = reader.readNext()) != null)
{
observations.add(Observations.instance(nextLine));
Expand All @@ -132,9 +132,9 @@ public static final Camtrap instance(String folderPath) throws FileNotFoundExcep
public Camtrap()
{
collectionID = "";
deployments = new Vector<Deployments>();
media = new Vector<Media>();
observations = new Vector<Observations>();
deployments = new ArrayList<Deployments>();
media = new ArrayList<Media>();
observations = new ArrayList<Observations>();
}

/**
Expand All @@ -146,11 +146,10 @@ public Camtrap()
*/
public final void setDeployments(String csvData) throws IOException, CsvValidationException
{
List<Deployments> deployments = new Vector<Deployments>();

CSVReader reader = new CSVReader(new StringReader(csvData));
String[] nextLine;

List<Deployments> deployments = new ArrayList<Deployments>(Math.toIntExact(reader.getLinesRead()));
while ((nextLine = reader.readNext()) != null)
{
deployments.add(Deployments.instance(nextLine));
Expand All @@ -168,11 +167,10 @@ public final void setDeployments(String csvData) throws IOException, CsvValidati
*/
public final void setMedia(String csvData) throws IOException, CsvValidationException
{
List<Media> media = new Vector<Media>();

CSVReader reader = new CSVReader(new StringReader(csvData));
String[] nextLine;

List<Media> media = new ArrayList<Media>(Math.toIntExact(reader.getLinesRead()));
while ((nextLine = reader.readNext()) != null)
{
media.add(Media.instance(nextLine));
Expand All @@ -190,11 +188,10 @@ public final void setMedia(String csvData) throws IOException, CsvValidationExce
*/
public final void setObservations(String csvData) throws IOException, CsvValidationException
{
List<Observations> observations = new Vector<Observations>();

CSVReader reader = new CSVReader(new StringReader(csvData));
String[] nextLine;

List<Observations> observations = new ArrayList<Observations>(Math.toIntExact(reader.getLinesRead()));
while ((nextLine = reader.readNext()) != null)
{
observations.add(Observations.instance(nextLine));
Expand Down
16 changes: 16 additions & 0 deletions Sanimal FX/src/main/java/model/query/S3Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.time.LocalDateTime;
import java.time.Month;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
Expand Down Expand Up @@ -75,6 +76,21 @@ public void addImageCollection(ImageCollection imageCollection)
this.collectionQuery.add(imageCollection);
}

/**
* Returns the list of configured image collections to search in
*/
public List<String> getCollectionIDs()
{
List<String> returnIDs = new ArrayList<String>();

for (ImageCollection oneCol: this.collectionQuery)
{
returnIDs.add(oneCol.getName());
}

return returnIDs;
}

/**
* Adds a given startYear to the query
*
Expand Down
52 changes: 45 additions & 7 deletions Sanimal FX/src/main/java/model/query/S3QueryExecute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import org.apache.commons.io.FilenameUtils;
import java.util.function.BiFunction;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.List;
import java.time.Instant;
import java.time.LocalDateTime;
Expand All @@ -32,7 +34,7 @@ public class S3QueryExecute
* @param collections the list of collections to check
* @return the result set of found images
*/
public static S3QueryResultSet executeQuery(S3QueryBuilder queryBuilder, final List<ImageCollection> collections)
public static S3QueryResultSet executeQuery(S3QueryBuilder queryBuilder, final List<ImageCollection> collections) throws InterruptedException, ExecutionException
{
S3QueryResultSet resultSet = new S3QueryResultSet();
List<S3QueryBuilderCondition> conditions = queryBuilder.getConditions();
Expand All @@ -45,22 +47,58 @@ public static S3QueryResultSet executeQuery(S3QueryBuilder queryBuilder, final L
return null;
}

// Define class to hold search results
class InternalResults
{
public String bucket;
public List<String> matches;
}

// Iterate through the collections and look at the metadata
List<CompletableFuture<List<InternalResults>>> allFutures = new ArrayList<CompletableFuture<List<InternalResults>>>();
for (ImageCollection oneCollection: collections)
{
if (!oneCollection.uploadsWereSynced())
{
System.out.println("executeQuery(): non-synched collection '" + oneCollection.getName() + "'");
}

List<CloudUploadEntry> uploads = oneCollection.getUploads();
for (CloudUploadEntry oneEntry: uploads)
CompletableFuture<List<InternalResults>> queryFuture = CompletableFuture.supplyAsync(() -> {
List<InternalResults> allMatches = new ArrayList<InternalResults>();
List<CloudUploadEntry> uploads = oneCollection.getUploads();
for (CloudUploadEntry oneEntry: uploads)
{
Camtrap metaData = oneEntry.getMetadata().getValue();
List<String> matches = S3QueryExecute.queryMatches(conditions, metaData, isCaseInsensitive);
if ((matches != null) && (matches.size() > 0))
{
InternalResults res = new InternalResults();
res.bucket = oneEntry.getBucket();
res.matches = matches;
allMatches.add(res);
}
}

return allMatches;
}
);

allFutures.add(queryFuture);
}

if (allFutures.size() > 0)
{
CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[allFutures.size()]));
combinedFuture.get();

for (CompletableFuture<List<InternalResults>> oneFuture: allFutures)
{
Camtrap metaData = oneEntry.getMetadata().getValue();
List<String> matches = S3QueryExecute.queryMatches(conditions, metaData, isCaseInsensitive);
if ((matches != null) && (matches.size() > 0))
for (InternalResults allMatches: oneFuture.get())
{
S3QueryExecute.addMatchesToResults(resultSet, oneEntry.getBucket(), matches, isDistinct);
if ((allMatches.matches != null) && (allMatches.matches.size() > 0))
{
S3QueryExecute.addMatchesToResults(resultSet, allMatches.bucket, allMatches.matches, isDistinct);
}
}
}
}
Expand Down
47 changes: 40 additions & 7 deletions Sanimal FX/src/main/java/model/s3/S3ConnectionManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import java.time.ZoneId;
import java.util.*;
import java.util.function.BiConsumer;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.security.InvalidParameterException;
Expand Down Expand Up @@ -1214,6 +1215,7 @@ public void retrieveAndInsertUploadList(ImageCollection collection, DoubleProper
if (uploadEntry != null)
{
uploadEntry.initFromJSON();

// Get the Camtrap data
uploadEntry.setMetadata(this.readRemoteCamtrap(collectionBucket, folder));

Expand Down Expand Up @@ -1326,16 +1328,34 @@ public List<String> performQuery(S3Query queryBuilder, final List<ImageCollectio
{
try
{
List<String> collFilterIDs = queryBuilder.getCollectionIDs();
List<CompletableFuture<Void>> allFutures = new ArrayList<CompletableFuture<Void>>();
long collFilterIDLen = collFilterIDs.size();
for (ImageCollection oneCollection: collections)
{
if (!oneCollection.uploadsWereSynced())
{
DoubleProperty progress = new SimpleDoubleProperty(0.0);
this.retrieveAndInsertUploadList(oneCollection, progress);
oneCollection.setUploadsWereSynced(true);
boolean loadColl = (collFilterIDLen == 0) || collFilterIDs.contains(oneCollection.getName());
if (loadColl)
{
CompletableFuture<Void> getFuture = CompletableFuture.supplyAsync(() -> {
DoubleProperty progress = new SimpleDoubleProperty(0.0);
this.retrieveAndInsertUploadList(oneCollection, progress);
oneCollection.setUploadsWereSynced(true);
return null;
}
);
allFutures.add(getFuture);
}
}
}

if (allFutures.size() > 0)
{
CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[allFutures.size()]));
combinedFuture.get();
}

S3QueryResultSet resultSet = S3QueryExecute.executeQuery(queryBuilder.build(), collections);

List<String> matchingFilePaths = new ArrayList<>();
Expand Down Expand Up @@ -1743,11 +1763,24 @@ private boolean objectExists(String bucket, String objectName)
* @return The contents of the file on S3's system as a string
*/
private String readRemoteFile(String bucket, String objectName)
{
return this.readRemoteFile(bucket, objectName, true);
}

/**
* Reads a file from S3 assuming a user is already logged in
*
* @param bucket The bucket to load the object from
* @param objectName The name of the Object to read
* @param checkExists Flag for performing an existence check before getting th object
* @return The contents of the file on S3's system as a string
*/
private String readRemoteFile(String bucket, String objectName, boolean checkExists)
{
try
{
// Ensure it exists
if (this.objectExists(bucket, objectName))
if (!checkExists || (checkExists && this.objectExists(bucket, objectName)))
{
return s3Client.getObjectAsString(bucket, objectName);
}
Expand Down Expand Up @@ -1791,15 +1824,15 @@ private Camtrap readRemoteCamtrap(String bucket, String prefix) throws IOExcepti
Camtrap metadata = new Camtrap();

String remotePath = String.join("/", prefix, Camtrap.CAMTRAP_DEPLOYMENTS_FILE);
String csvData = this.readRemoteFile(bucket, remotePath);
String csvData = this.readRemoteFile(bucket, remotePath, false);
metadata.setDeployments(csvData);

remotePath = String.join("/", prefix, Camtrap.CAMTRAP_MEDIA_FILE);
csvData = this.readRemoteFile(bucket, remotePath);
csvData = this.readRemoteFile(bucket, remotePath, false);
metadata.setMedia(csvData);

remotePath = String.join("/", prefix, Camtrap.CAMTRAP_OBSERVATIONS_FILE);
csvData = this.readRemoteFile(bucket, remotePath);
csvData = this.readRemoteFile(bucket, remotePath, false);
metadata.setObservations(csvData);

return metadata;
Expand Down
Binary file modified Sanimal FX/src/main/resources/images/mainMenu/CCTlogo.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.

0 comments on commit cd58b9b

Please sign in to comment.