Skip to content

Commit

Permalink
Implement Publish v3
Browse files Browse the repository at this point in the history 
- Re-run notify if Tus returns error "upload is still in process".
- Poll status every 10 seconds (12 times) while response is 202.
- Use horizontal progress bar with determinate progress while Tus is
  uploading. This lets there be a meaningful indicator while uploading a
  large file.

The "Check Status" button (used when the retry count has been exceeded
polling status) is only shown in the Publish Form, since it won't
navigate away until the publish is successful.

Fix: #365
  • Loading branch information
@ktprograms
ktprograms committed Oct 29, 2022
1 parent 4d5520a commit 75a7ddb
Show file tree
Hide file tree
Showing 4 changed files with 182 additions and 56 deletions.
219 changes: 165 additions & 54 deletions app/src/main/java/com/odysee/app/tasks/claim/TusPublishTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import android.os.AsyncTask;
import android.view.View;
import android.widget.ProgressBar;

import com.odysee.app.exceptions.LbryResponseException;
import com.odysee.app.model.Claim;
Expand All @@ -14,13 +15,9 @@

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;

Expand All @@ -29,27 +26,34 @@
import io.tus.java.client.TusExecutor;
import io.tus.java.client.TusUpload;
import io.tus.java.client.TusUploader;
import lombok.Getter;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;

// Due to the TusExecutor callback, result is saved in claimResult
// instead of returning through AsyncTask methods.
public class TusPublishTask extends AsyncTask<Void, Void, Void> {
public class TusPublishTask extends AsyncTask<Void, Integer, Void> {
private static final int NOTIFY_RETRY_INTERVAL = 5000;
private static final int STATUS_RETRY_COUNT = 12;
private static final int STATUS_RETRY_INTERVAL = 10000;

private final Claim claim;
private final String filePath;
private final View progressView;
private final String uploadUrl;
private final ProgressBar progressView;
private final String authToken;
private final ClaimResultHandler handler;

private Exception error;
private Claim claimResult;

public TusPublishTask(Claim claim, String filePath, View progressView,
String authToken, ClaimResultHandler handler) {
public TusPublishTask(Claim claim, String filePath, String uploadUrl,
ProgressBar progressView, String authToken, ClaimResultHandler handler) {
this.claim = claim;
this.filePath = filePath;
this.uploadUrl = uploadUrl;
this.progressView = progressView;
this.authToken = authToken;
this.handler = handler;
Expand All @@ -65,9 +69,14 @@ protected void onPreExecute() {

@Override
protected Void doInBackground(Void... voids) {
if (!Helper.isNullOrEmpty(uploadUrl)) {
sendStatusRequest(uploadUrl, STATUS_RETRY_COUNT);
return null;
}

try {
TusClient client = new TusClient();
client.setUploadCreationURL(new URL("https://api.na-backend.odysee.com/api/v2/publish/"));
client.setUploadCreationURL(new URL("https://api.na-backend.odysee.com/api/v3/publish/"));
// TODO: enableResuming
Map<String, String> headers = new HashMap<>();
headers.put("X-Lbry-Auth-Token", authToken);
Expand All @@ -81,52 +90,16 @@ protected Void doInBackground(Void... voids) {
protected void makeAttempt() throws ProtocolException, IOException {
TusUploader uploader = client.createUpload(upload);
uploader.setChunkSize(1024 * 1024);
while (uploader.uploadChunk() > -1);
do {
long total = upload.getSize();
long bytesUploaded = uploader.getOffset();
int progress = (int) ((double) bytesUploaded / total * 100);
publishProgress(progress);
} while (uploader.uploadChunk() > -1);
uploader.finish();

URL notifyURL = new URL(uploader.getUploadURL().toString() + "/notify");
JSONObject requestBody = new JSONObject();
try {
Map<String, Object> options = Helper.buildPublishOptions(claim);

JSONObject params = Lbry.buildJsonParams(options);
long counter = Double.valueOf(System.currentTimeMillis() / 1000.0).longValue();
requestBody.put("jsonrpc", "2.0");
requestBody.put("method", Lbry.METHOD_PUBLISH);
requestBody.put("params", params);
requestBody.put("counter", counter);
} catch (JSONException ex) {
error = ex;
return;
}

RequestBody body = RequestBody.create(requestBody.toString(), Helper.JSON_MEDIA_TYPE);
Request.Builder requestBuilder = new Request.Builder().url(notifyURL).post(body);
requestBuilder.addHeader("X-Lbry-Auth-Token", authToken);
requestBuilder.addHeader("Tus-Resumable", "1.0.0");
Request request = requestBuilder.build();
OkHttpClient client = new OkHttpClient.Builder()
.writeTimeout(300, TimeUnit.SECONDS)
.readTimeout(300, TimeUnit.SECONDS)
.build();

try {
Response response = client.newCall(request).execute();
JSONObject result = (JSONObject) Lbry.parseResponse(response);
if (result.has("outputs")) {
JSONArray outputs = result.getJSONArray("outputs");
for (int i = 0; i < outputs.length(); i++) {
JSONObject output = outputs.getJSONObject(i);
if (output.has("claim_id") && output.has("claim_op")) {
claimResult = Claim.claimFromOutput(output);
break;
}
}
}
} catch (IOException | LbryResponseException | ClassCastException
| JSONException ex) {
error = ex;
}
publishProgress(-1);
makeNotifyRequest(uploader.getUploadURL().toString());
}
};
executor.makeAttempts();
Expand All @@ -137,6 +110,133 @@ protected void makeAttempt() throws ProtocolException, IOException {
return null;
}

@Override
protected void onProgressUpdate(Integer... values) {
int progress = values[0];
if (progress >= 0) {
progressView.setIndeterminate(false);
progressView.setProgress(progress);
} else { // -1 = not uploading, so show indeterminate
progressView.setIndeterminate(true);
}
}

void makeNotifyRequest(String uploadUrl) {
URL notifyUrl;
JSONObject requestBody = new JSONObject();
try {
notifyUrl = new URL(uploadUrl + "/notify");

Map<String, Object> options = Helper.buildPublishOptions(claim);
JSONObject params = Lbry.buildJsonParams(options);
long counter = Double.valueOf(System.currentTimeMillis() / 1000.0).longValue();
requestBody.put("jsonrpc", "2.0");
requestBody.put("method", Lbry.METHOD_PUBLISH);
requestBody.put("params", params);
requestBody.put("counter", counter);
} catch (JSONException | MalformedURLException ex) {
error = ex;
return;
}

RequestBody body = RequestBody.create(requestBody.toString(), Helper.JSON_MEDIA_TYPE);
Request.Builder requestBuilder = new Request.Builder().url(notifyUrl).post(body);
requestBuilder.addHeader("X-Lbry-Auth-Token", authToken);
requestBuilder.addHeader("Tus-Resumable", "1.0.0");
Request request = requestBuilder.build();
OkHttpClient client = new OkHttpClient.Builder()
.writeTimeout(300, TimeUnit.SECONDS)
.readTimeout(300, TimeUnit.SECONDS)
.build();

try {
Response response = client.newCall(request).execute();

try {
Lbry.parseResponse(response);
} catch (LbryResponseException ex) {
if (ex.getMessage() != null &&
ex.getMessage().equalsIgnoreCase("upload is still in process")) {
Thread.sleep(NOTIFY_RETRY_INTERVAL);
makeNotifyRequest(uploadUrl);
return;
}
}

sendStatusRequest(uploadUrl, STATUS_RETRY_COUNT);
} catch (IOException | InterruptedException ex) {
error = ex;
}
}

void sendStatusRequest(String uploadUrl, int retryCount) {
try {
URL statusUrl = new URL(uploadUrl + "/status");
Request.Builder requestBuilder = new Request.Builder().url(statusUrl).get();
requestBuilder.addHeader("Content-Type", "application/json");
requestBuilder.addHeader("X-Lbry-Auth-Token", authToken);
requestBuilder.addHeader("Tus-Resumable", "1.0.0");
Request request = requestBuilder.build();
OkHttpClient client = new OkHttpClient.Builder()
.writeTimeout(300, TimeUnit.SECONDS)
.readTimeout(300, TimeUnit.SECONDS)
.build();

Response response = client.newCall(request).execute();

switch (response.code()) {
case 200:
JSONObject result = (JSONObject) Lbry.parseResponse(response);
if (result.has("outputs")) {
JSONArray outputs = result.getJSONArray("outputs");
for (int i = 0; i < outputs.length(); i++) {
JSONObject output = outputs.getJSONObject(i);
if (output.has("claim_id") && output.has("claim_op")) {
claimResult = Claim.claimFromOutput(output);
break;
}
}
}
break;

case 202:
if (retryCount > 0) {
Thread.sleep(STATUS_RETRY_INTERVAL);
sendStatusRequest(uploadUrl, retryCount - 1);
} else {
error = new CheckStatusException(
"The file is still being processed. Check back later after a few minutes.",
uploadUrl
);
}
break;

case 403:
case 404:
error = new LbryResponseException("The upload does not exist");
break;

case 409:
// Get SDK error from response
try {
Lbry.parseResponse(response);
} catch (LbryResponseException ex) {
if (ex.getMessage() != null) {
error = new LbryResponseException("Failed to process the uploaded file: " + ex.getMessage());
} else {
error = new LbryResponseException("Failed to process the uploaded file");
}
}
break;

default:
error = new LbryResponseException("Unexpected error: " + response.code());
}
} catch (IOException | InterruptedException | LbryResponseException | JSONException ex) {
error = ex;
}
}

@Override
protected void onPostExecute(Void unused) {
Helper.setViewVisibility(progressView, View.GONE);
Expand All @@ -148,4 +248,15 @@ protected void onPostExecute(Void unused) {
}
}
}

// Not actually an exception, just a handy type for returning a value.
// Used when status request returns 202 and it's exceeded retry count.
public static class CheckStatusException extends Exception {
@Getter
private final String uploadUrl;
public CheckStatusException(String message, String uploadUrl) {
super(message);
this.uploadUrl = uploadUrl;
}
}
}
13 changes: 12 additions & 1 deletion app/src/main/java/com/odysee/app/ui/publish/PublishFormFragment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public class PublishFormFragment extends BaseFragment implements
private boolean editMode;
@Getter
private boolean saveInProgress;
private String pendingStatusUploadUrl;
private String currentFilter;
private boolean publishFileChecked;
private boolean fetchingChannels;
Expand Down Expand Up @@ -1268,12 +1269,22 @@ public void onSuccess(Claim claimResult) {

@Override
public void onError(Exception error) {
if (error instanceof TusPublishTask.CheckStatusException) {
pendingStatusUploadUrl = ((TusPublishTask.CheckStatusException) error).getUploadUrl();
postSave();
showMessage(error.getMessage());
Helper.setViewText(buttonPublish, R.string.check_status);
return;
}

showError(error.getMessage());
postSave();
}
};

if (!editMode) {
TusPublishTask task = new TusPublishTask(claim, finalFilePath, progressPublish, Lbryio.AUTH_TOKEN, handler);
TusPublishTask task = new TusPublishTask(claim, finalFilePath,
pendingStatusUploadUrl, progressPublish, Lbryio.AUTH_TOKEN, handler);
task.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR);
} else {
PublishClaimTask task = new PublishClaimTask(claim, progressPublish, Lbryio.AUTH_TOKEN, handler);
Expand Down
5 changes: 4 additions & 1 deletion app/src/main/res/layout/fragment_publish_form.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,10 +631,13 @@
<ProgressBar
android:id="@+id/publish_form_publishing"
android:layout_centerVertical="true"
android:layout_width="20dp"
android:layout_width="0dp"
android:layout_height="20dp"
android:layout_marginStart="16dp"
android:layout_marginEnd="16dp"
android:layout_toStartOf="@id/publish_form_publish_button"
android:layout_toEndOf="@id/publish_form_cancel"
style="?android:attr/progressBarStyleHorizontal"
android:visibility="gone" />

<com.google.android.material.button.MaterialButton
Expand Down
1 change: 1 addition & 0 deletions app/src/main/res/values/strings.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@
</plurals>

<!-- Publish -->
<string name="check_status">Check Status</string>
<string name="no_publishes_created">It looks like you have not published content to LBRY yet.</string>
<string name="record">Record</string>
<string name="take_photo">Take a Photo</string>
Expand Down

0 comments on commit 75a7ddb

Please sign in to comment.