Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP BAD Joel/leo misc apis #8872

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.broadinstitute.dsde.workbench.client.leonardo.model.DiskStatus;
import org.broadinstitute.dsde.workbench.client.leonardo.model.GetRuntimeResponse;
import org.broadinstitute.dsde.workbench.client.leonardo.model.ListPersistentDiskResponse;
import org.pmiops.workbench.billing.FreeTierBillingService;
import org.pmiops.workbench.config.WorkbenchConfig;
Expand All @@ -29,8 +29,6 @@
import org.pmiops.workbench.firecloud.FireCloudService;
import org.pmiops.workbench.firecloud.FirecloudTransforms;
import org.pmiops.workbench.legacy_leonardo_client.ApiException;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoGetRuntimeResponse;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoListRuntimeResponse;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoRuntimeStatus;
import org.pmiops.workbench.leonardo.LeonardoApiClient;
import org.pmiops.workbench.mail.MailService;
Expand Down Expand Up @@ -117,21 +115,22 @@ public ResponseEntity<Void> deleteOldRuntimes() {
final Duration maxAge = Duration.ofDays(config.firecloud.notebookRuntimeMaxAgeDays);
final Duration idleMaxAge = Duration.ofDays(config.firecloud.notebookRuntimeIdleMaxAgeDays);

final List<LeonardoListRuntimeResponse> listRuntimeResponses =
leonardoApiClient.listRuntimesAsService();
final List<org.broadinstitute.dsde.workbench.client.leonardo.model.ListRuntimeResponse>
listRuntimeResponses = leonardoApiClient.listRuntimesAsService();

int idles = 0;
int activeDeletes = 0;
int unusedDeletes = 0;
for (LeonardoListRuntimeResponse listRuntimeResponse : listRuntimeResponses) {
for (org.broadinstitute.dsde.workbench.client.leonardo.model.ListRuntimeResponse
listRuntimeResponse : listRuntimeResponses) {
final String googleProject =
leonardoMapper.toGoogleProject(listRuntimeResponse.getCloudContext());
final String runtimeId =
String.format("%s/%s", googleProject, listRuntimeResponse.getRuntimeName());

// Refetch the runtime to ensure freshness,
// as this iteration may take some time.
final LeonardoGetRuntimeResponse runtime =
final GetRuntimeResponse runtime =
leonardoApiClient.getRuntimeAsService(
googleProject, listRuntimeResponse.getRuntimeName());

Expand Down Expand Up @@ -330,9 +329,12 @@ private boolean isDiskAttached(ListPersistentDiskResponse diskResponse, String g
final String diskName = diskResponse.getName();

if (leonardoMapper.toApiListDisksResponse(diskResponse).isGceRuntime()) {
return leonardoApiClient.listRuntimesByProjectAsService(googleProject).stream()
.flatMap(runtime -> Stream.ofNullable(runtime.getDiskConfig()))
.anyMatch(diskConfig -> diskName.equals(diskConfig.getName()));
return
// leonardoApiClient.listRuntimesByProjectAsService(googleProject).stream()
// .flatMap(runtime -> Stream.ofNullable(runtime.getDiskConfig()))
// .anyMatch(diskConfig -> diskName.equals(diskConfig.getName()));
// TODO VERY WRONG
true;
} else {
return leonardoApiClient.listAppsInProjectAsService(googleProject).stream()
.anyMatch(userAppEnvironment -> diskName.equals(userAppEnvironment.getDiskName()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.broadinstitute.dsde.workbench.client.leonardo.model.ClusterError;
import org.broadinstitute.dsde.workbench.client.leonardo.model.ClusterStatus;
import org.broadinstitute.dsde.workbench.client.leonardo.model.GetRuntimeResponse;
import org.broadinstitute.dsde.workbench.client.leonardo.model.ListRuntimeResponse;
import org.pmiops.workbench.config.WorkbenchConfig;
import org.pmiops.workbench.db.model.DbUser;
import org.pmiops.workbench.db.model.DbWorkspace;
import org.pmiops.workbench.exceptions.BadRequestException;
import org.pmiops.workbench.exceptions.NotFoundException;
import org.pmiops.workbench.interactiveanalysis.InteractiveAnalysisService;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoClusterError;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoGetRuntimeResponse;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoListRuntimeResponse;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoRuntimeStatus;
import org.pmiops.workbench.leonardo.LeonardoApiClient;
import org.pmiops.workbench.leonardo.LeonardoApiHelper;
import org.pmiops.workbench.leonardo.PersistentDiskUtils;
Expand Down Expand Up @@ -92,9 +92,9 @@ public ResponseEntity<Runtime> getRuntime(String workspaceNamespace) {
DbWorkspace dbWorkspace = workspaceService.lookupWorkspaceByNamespace(workspaceNamespace);
String googleProject = dbWorkspace.getGoogleProject();
try {
LeonardoGetRuntimeResponse leoRuntimeResponse =
GetRuntimeResponse leoRuntimeResponse =
leonardoNotebooksClient.getRuntime(googleProject, user.getRuntimeName());
if (LeonardoRuntimeStatus.ERROR.equals(leoRuntimeResponse.getStatus())) {
if (ClusterStatus.ERROR.equals(leoRuntimeResponse.getStatus())) {
log.warning(
String.format(
"Observed Leonardo runtime with unexpected error status:\n%s",
Expand All @@ -106,7 +106,7 @@ public ResponseEntity<Runtime> getRuntime(String workspaceNamespace) {
}
}

private String formatRuntimeErrors(@Nullable List<LeonardoClusterError> errors) {
private String formatRuntimeErrors(@Nullable List<ClusterError> errors) {
if (errors == null || errors.isEmpty()) {
return "no error messages";
}
Expand All @@ -116,7 +116,7 @@ private String formatRuntimeErrors(@Nullable List<LeonardoClusterError> errors)
}

private Runtime getOverrideFromListRuntimes(String googleProject) {
Optional<LeonardoListRuntimeResponse> mostRecentRuntimeMaybe =
Optional<ListRuntimeResponse> mostRecentRuntimeMaybe =
leonardoNotebooksClient.listRuntimesByProject(googleProject, true).stream()
.min(
(a, b) -> {
Expand All @@ -136,7 +136,7 @@ private Runtime getOverrideFromListRuntimes(String googleProject) {
return bCreatedDate.compareTo(aCreatedDate);
});

LeonardoListRuntimeResponse mostRecentRuntime =
ListRuntimeResponse mostRecentRuntime =
mostRecentRuntimeMaybe.orElseThrow(NotFoundException::new);

@SuppressWarnings("unchecked")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,35 @@

import java.util.List;
import java.util.Map;
import org.broadinstitute.dsde.workbench.client.leonardo.model.GetRuntimeResponse;
import org.broadinstitute.dsde.workbench.client.leonardo.model.ListPersistentDiskResponse;
import org.broadinstitute.dsde.workbench.client.leonardo.model.ListRuntimeResponse;
import org.pmiops.workbench.db.model.DbWorkspace;
import org.pmiops.workbench.exceptions.WorkbenchException;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoGetRuntimeResponse;
import org.pmiops.workbench.legacy_leonardo_client.model.LeonardoListRuntimeResponse;
import org.pmiops.workbench.model.CreateAppRequest;
import org.pmiops.workbench.model.Runtime;
import org.pmiops.workbench.model.UserAppEnvironment;
import org.pmiops.workbench.notebooks.model.StorageLink;

/**
* Encapsulate Leonardo's Notebooks API interaction details and provide a simple/mockable interface
* for internal use.
* Encapsulate Leonardo's API interaction details and provide a simple/mockable interface for
* internal use.
*/
public interface LeonardoApiClient {

//
// Runtime endpoints (legacy client)
//

/**
* lists all runtimes in the environment as the appengine SA, to be used only for admin operations
*/
List<LeonardoListRuntimeResponse> listRuntimesAsService();
List<ListRuntimeResponse> listRuntimesAsService();

/** lists all notebook runtimes as the appengine SA, to be used only for admin operations */
List<LeonardoListRuntimeResponse> listRuntimesByProjectAsService(String googleProject);
List<ListRuntimeResponse> listRuntimesByProjectAsService(String googleProject);

List<LeonardoListRuntimeResponse> listRuntimesByProject(
String googleProject, boolean includeDeleted);
List<ListRuntimeResponse> listRuntimesByProject(String googleProject, boolean includeDeleted);

/**
* Creates a notebooks runtime owned by the current authenticated user.
Expand All @@ -46,7 +50,7 @@ void deleteRuntime(String googleProject, String runtimeName, Boolean deleteDisk)
throws WorkbenchException;

/** Retrieves a notebook runtime as the appengine SA, to be used only for admin operations */
LeonardoGetRuntimeResponse getRuntimeAsService(String googleProject, String runtimeName)
GetRuntimeResponse getRuntimeAsService(String googleProject, String runtimeName)
throws WorkbenchException;

/** Deletes a notebook runtime as the appengine SA, to be used only for admin operations */
Expand All @@ -59,8 +63,11 @@ LeonardoGetRuntimeResponse getRuntimeAsService(String googleProject, String runt
int stopAllUserRuntimesAsService(String userEmail) throws WorkbenchException;

/** Gets information about a notebook runtime */
LeonardoGetRuntimeResponse getRuntime(String googleProject, String runtimeName)
throws WorkbenchException;
GetRuntimeResponse getRuntime(String googleProject, String runtimeName) throws WorkbenchException;

//
// Welder endpoints - "notebooks" client
//

/** Send files over to notebook runtime */
void localizeForRuntime(String googleProject, String runtimeName, Map<String, String> fileList)
Expand All @@ -78,6 +85,10 @@ void localizeForApp(String googleProject, String appName, Map<String, String> fi
StorageLink createStorageLinkForApp(
String googleProject, String appName, StorageLink storageLink);

//
// Disk endpoints
//

/** Deletes a persistent disk */
void deletePersistentDisk(String googleProject, String diskName) throws WorkbenchException;

Expand All @@ -93,6 +104,16 @@ void updatePersistentDisk(String googleProject, String diskName, Integer diskSiz
List<ListPersistentDiskResponse> listPersistentDiskByProjectCreatedByCreator(
String googleProject);

/** List all persistent disks */
List<ListPersistentDiskResponse> listDisksAsService();

/** List all persistent disks in google project */
List<ListPersistentDiskResponse> listDisksByProjectAsService(String googleProject);

//
// Apps endpoints
//

/**
* Creates Leonardo app owned by the current authenticated user.
*
Expand All @@ -102,6 +123,8 @@ List<ListPersistentDiskResponse> listPersistentDiskByProjectCreatedByCreator(
void createApp(CreateAppRequest createAppRequest, DbWorkspace dbWorkspace)
throws WorkbenchException;

int deleteUserAppsAsService(String userEmail);

/**
* Lists all apps the user creates in the given workspace GCP project
*
Expand All @@ -121,18 +144,18 @@ void createApp(CreateAppRequest createAppRequest, DbWorkspace dbWorkspace)
void deleteApp(String appName, DbWorkspace dbWorkspace, boolean deleteDisk)
throws WorkbenchException;

//
// Status endpoints
//

/**
* @return true if Leonardo service is okay, false otherwise.
*/
boolean getLeonardoStatus();

int deleteUserAppsAsService(String userEmail);

/** List all persistent disks */
List<ListPersistentDiskResponse> listDisksAsService();

/** List all persistent disks in google project */
List<ListPersistentDiskResponse> listDisksByProjectAsService(String googleProject);
//
// Resources endpoints
//

void deleteAllResources(String googleProject, boolean deleteDisk);
}
Loading