Skip to content

Commit

Permalink
Merge branch 'deephaven:main' into docs-ci-condition-on-latest-link
Browse files Browse the repository at this point in the history
  • Loading branch information
@stanbrub
stanbrub authored Jun 24, 2024
2 parents 4726fe4 + 31a9895 commit 5fbc8a1
Show file tree
Hide file tree
Showing 13 changed files with 563 additions and 258 deletions.
4 changes: 2 additions & 2 deletions buildSrc/src/main/groovy/Classpaths.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,10 @@ class Classpaths {
static final String HADOOP_VERSION = '3.4.0'

static final String ICEBERG_GROUP = 'org.apache.iceberg'
static final String ICEBERG_VERSION = '1.5.0'
static final String ICEBERG_VERSION = '1.5.2'

static final String AWSSDK_GROUP = 'software.amazon.awssdk'
static final String AWSSDK_VERSION = '2.23.19'
static final String AWSSDK_VERSION = '2.24.5'

static final String TESTCONTAINER_GROUP = 'org.testcontainers'
static final String TESTCONTAINER_VERSION = '1.19.4'
Expand Down
193 changes: 0 additions & 193 deletions engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2931,199 +2931,6 @@ public static int minuteOfHour(@Nullable final ZonedDateTime dateTime) {
return dateTime.getMinute();
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// ↓↓↓↓↓↓↓ THE METHODS BELOW ARE DEPRECATED AND WILL BE REMOVED SOON ↓↓↓↓↓↓↓
///////////////////////////////////////////////////////////////////////////////////////////////////

/**
* Returns the number of nanoseconds that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param instant time
* @param timeZone time zone
* @return {@link QueryConstants#NULL_LONG} if either input is {@code null}; otherwise, number of nanoseconds that
* have elapsed since the top of the day
* @deprecated Use {@link #nanosOfDay(Instant, ZoneId, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static long nanosOfDay(@Nullable final Instant instant, @Nullable final ZoneId timeZone) {
return nanosOfDay(instant, timeZone, false);
}

/**
* Returns the number of nanoseconds that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param dateTime time
* @return {@link QueryConstants#NULL_LONG} if either input is {@code null}; otherwise, number of nanoseconds that
* have elapsed since the top of the day
* @deprecated Use {@link #nanosOfDay(ZonedDateTime, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static long nanosOfDay(@Nullable final ZonedDateTime dateTime) {
return nanosOfDay(dateTime, false);
}

/**
* Returns the number of milliseconds that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param instant time
* @param timeZone time zone
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of milliseconds that
* have elapsed since the top of the day
* @deprecated Use {@link #millisOfDay(Instant, ZoneId, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int millisOfDay(@Nullable final Instant instant, @Nullable final ZoneId timeZone) {
return millisOfDay(instant, timeZone, false);
}

/**
* Returns the number of milliseconds that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param dateTime time
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of milliseconds that
* have elapsed since the top of the day
* @deprecated Use {@link #millisOfDay(ZonedDateTime, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int millisOfDay(@Nullable final ZonedDateTime dateTime) {
return millisOfDay(dateTime, false);
}

/**
* Returns the number of seconds that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param instant time
* @param timeZone time zone
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of seconds that have
* elapsed since the top of the day
* @deprecated Use {@link #secondOfDay(Instant, ZoneId, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int secondOfDay(@Nullable final Instant instant, @Nullable final ZoneId timeZone) {
return secondOfDay(instant, timeZone, false);
}

/**
* Returns the number of seconds that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param dateTime time
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of seconds that have
* elapsed since the top of the day
* @deprecated Use {@link #secondOfDay(ZonedDateTime, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int secondOfDay(@Nullable final ZonedDateTime dateTime) {
return secondOfDay(dateTime, false);
}

/**
* Returns the number of minutes that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param instant time
* @param timeZone time zone
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of minutes that have
* elapsed since the top of the day
* @deprecated Use {@link #minuteOfDay(Instant, ZoneId, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int minuteOfDay(@Nullable final Instant instant, @Nullable final ZoneId timeZone) {
return minuteOfDay(instant, timeZone, false);
}

/**
* Returns the number of minutes that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param dateTime time
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of minutes that have
* elapsed since the top of the day
* @deprecated Use {@link #minuteOfDay(ZonedDateTime, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int minuteOfDay(@Nullable final ZonedDateTime dateTime) {
return minuteOfDay(dateTime, false);
}

/**
* Returns the number of hours that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param instant time
* @param timeZone time zone
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of hours that have
* elapsed since the top of the day
* @deprecated Use {@link #hourOfDay(Instant, ZoneId, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int hourOfDay(@Nullable final Instant instant, @Nullable final ZoneId timeZone) {
return hourOfDay(instant, timeZone, false);
}

/**
* Returns the number of hours that have elapsed since the top of the day.
* <p>
* On days when daylight savings time events occur, results may be different from what is expected based upon the
* local time. For example, on daylight savings time change days, 9:30AM may be earlier or later in the day based
* upon if the daylight savings time adjustment is forwards or backwards.
*
* @param dateTime time
* @return {@link QueryConstants#NULL_INT} if either input is {@code null}; otherwise, number of hours that have
* elapsed since the top of the day
* @deprecated Use {@link #hourOfDay(ZonedDateTime, boolean)} instead. To be removed soon.
*/
@Deprecated
@ScriptApi
public static int hourOfDay(@Nullable final ZonedDateTime dateTime) {
return hourOfDay(dateTime, false);
}

///////////////////////////////////////////////////////////////////////////////////////////////////
// ↑↑↑↑↑↑↑ THE METHODS ABOVE ARE DEPRECATED AND WILL BE REMOVED SOON ↑↑↑↑↑↑↑
///////////////////////////////////////////////////////////////////////////////////////////////////

/**
* Returns the number of nanoseconds that have elapsed since the start of the day.
*
Expand Down
4 changes: 3 additions & 1 deletion extensions/iceberg/s3/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ dependencies {

implementation project(':extensions-s3')
implementation "org.apache.iceberg:iceberg-aws"
runtimeOnly "org.apache.iceberg:iceberg-aws-bundle"

Classpaths.inheritAWSSDK(project)
runtimeOnly "software.amazon.awssdk:sts"
runtimeOnly "software.amazon.awssdk:glue"

Classpaths.inheritTestContainers(project)

Expand Down
48 changes: 47 additions & 1 deletion extensions/iceberg/s3/src/main/java/io/deephaven/iceberg/util/IcebergToolsS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.aws.AwsClientProperties;
import org.apache.iceberg.aws.glue.GlueCatalog;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.rest.RESTCatalog;
Expand All @@ -23,6 +24,20 @@
public class IcebergToolsS3 extends IcebergTools {
private static final String S3_FILE_IO_CLASS = "org.apache.iceberg.aws.s3.S3FileIO";

/**
* Create an Iceberg catalog adapter for a REST catalog backed by S3 storage. If {@code null} is provided for a
* value, the system defaults will be used.
*
* @param name the name of the catalog; if omitted, the catalog URI will be used to generate a name
* @param catalogURI the URI of the Iceberg REST catalog
* @param warehouseLocation the location of the S3 datafiles backing the catalog
* @param region the AWS region; if omitted, system defaults will be used
* @param accessKeyId the AWS access key ID; if omitted, system defaults will be used
* @param secretAccessKey the AWS secret access key; if omitted, system defaults will be used
* @param endpointOverride the S3 endpoint override; this is useful for testing with a S3-compatible local service
* such as MinIO or LocalStack
* @return the Iceberg catalog adapter
*/
public static IcebergCatalogAdapter createS3Rest(
@Nullable final String name,
@NotNull final String catalogURI,
Expand Down Expand Up @@ -53,7 +68,6 @@ public static IcebergCatalogAdapter createS3Rest(
properties.put(S3FileIOProperties.ENDPOINT, endpointOverride);
}

// TODO: create a FileIO interface wrapping the Deephaven S3SeekableByteChannel/Provider
final FileIO fileIO = CatalogUtil.loadFileIO(S3_FILE_IO_CLASS, properties, null);

final String catalogName = name != null ? name : "IcebergCatalog-" + catalogURI;
Expand All @@ -62,4 +76,36 @@ public static IcebergCatalogAdapter createS3Rest(
return new IcebergCatalogAdapter(catalog, fileIO);
}

/**
* Create an Iceberg catalog adapter for an AWS Glue catalog. System defaults will be used to populate the region
* and credentials. These can be configured by following
* <a href="https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html">AWS Authentication and
* access credentials</a> guide.
*
* @param name the name of the catalog; if omitted, the catalog URI will be used to generate a name
* @param catalogURI the URI of the AWS Glue catalog
* @param warehouseLocation the location of the S3 datafiles backing the catalog
* @return the Iceberg catalog adapter
*/
public static IcebergCatalogAdapter createGlue(
@Nullable final String name,
@NotNull final String catalogURI,
@NotNull final String warehouseLocation) {

// Set up the properties map for the Iceberg catalog
final Map<String, String> properties = new HashMap<>();

final GlueCatalog catalog = new GlueCatalog();

properties.put(CatalogProperties.CATALOG_IMPL, catalog.getClass().getName());
properties.put(CatalogProperties.URI, catalogURI);
properties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation);

final FileIO fileIO = CatalogUtil.loadFileIO(S3_FILE_IO_CLASS, properties, null);

final String catalogName = name != null ? name : "IcebergCatalog-" + catalogURI;
catalog.initialize(catalogName, properties);

return new IcebergCatalogAdapter(catalog, fileIO);
}
}
Loading

0 comments on commit 5fbc8a1

Please sign in to comment.