diff --git a/main.ts b/main.ts index 7b9fd97f..d9ab083f 100644 --- a/main.ts +++ b/main.ts @@ -238,84 +238,39 @@ export default class DigitalGarden extends Plugin { imagesToDelete.length, ); - let errorFiles = 0; - let errorDeleteFiles = 0; - let errorDeleteImage = 0; - new Notice( `Publishing ${filesToPublish.length} notes, deleting ${filesToDelete.length} notes and ${imagesToDelete.length} images. See the status bar in lower right corner for progress.`, 8000, ); - for (const file of filesToPublish) { - try { - statusBar.increment(); - await publisher.publish(file); - } catch { - errorFiles++; - - new Notice( - `Unable to publish note ${file.file.name}, skipping it.`, - ); - } - } - - for (const filePath of filesToDelete) { - try { - statusBar.increment(); - - // TODO: include sha from file.remoteHash to make faster! - await publisher.deleteNote( - filePath.path, - filePath.sha, - ); - } catch { - errorDeleteFiles++; + await publisher.publishBatch(filesToPublish); + statusBar.incrementMultiple(filesToPublish.length); - new Notice( - `Unable to delete note ${filePath}, skipping it.`, - ); - } - } - - for (const filePath of imagesToDelete) { - try { - statusBar.increment(); - - await publisher.deleteImage( - filePath.path, - filePath.sha, - ); - } catch { - errorDeleteImage++; + await publisher.deleteBatch( + filesToDelete.map((f) => f.path), + ); + statusBar.incrementMultiple(filesToDelete.length); - new Notice( - `Unable to delete image ${filePath}, skipping it.`, - ); - } - } + await publisher.deleteBatch( + imagesToDelete.map((f) => f.path), + ); + statusBar.incrementMultiple(imagesToDelete.length); statusBar.finish(8000); new Notice( - `Successfully published ${ - filesToPublish.length - errorFiles - } notes to your garden.`, + `Successfully published ${filesToPublish.length} notes to your garden.`, ); if (filesToDelete.length > 0) { new Notice( - `Successfully deleted ${ - filesToDelete.length - errorDeleteFiles - } notes from your garden.`, + `Successfully deleted ${filesToDelete.length} notes from your garden.`, ); } if (imagesToDelete.length > 0) { new Notice( - `Successfully deleted ${ - imagesToDelete.length - errorDeleteImage - } images from your garden.`, + `Successfully deleted ${imagesToDelete.length} images from your garden.`, ); } } catch (e) { diff --git a/manifest-beta.json b/manifest-beta.json index 0812ada2..bad5ac40 100644 --- a/manifest-beta.json +++ b/manifest-beta.json @@ -1,7 +1,7 @@ { "id": "digitalgarden", "name": "Digital Garden", - "version": "2.56.2", + "version": "2.57.0", "minAppVersion": "0.12.0", "description": "Publish your notes to the web for others to enjoy. For free.", "author": "Ole Eskild Steensen", diff --git a/src/publisher/Publisher.ts b/src/publisher/Publisher.ts index 6562cd13..12d0387a 100644 --- a/src/publisher/Publisher.ts +++ b/src/publisher/Publisher.ts @@ -99,7 +99,7 @@ export default class Publisher { return await this.delete(path, sha); } /** If provided with sha, garden connection does not need to get it seperately! */ - async delete(path: string, sha?: string): Promise { + public async delete(path: string, sha?: string): Promise { this.validateSettings(); const userGardenConnection = new RepositoryConnection({ @@ -115,7 +115,7 @@ export default class Publisher { return !!deleted; } - async publish(file: CompiledPublishFile): Promise { + public async publish(file: CompiledPublishFile): Promise { if (!isPublishFrontmatterValid(file.frontmatter)) { return false; } @@ -133,7 +133,55 @@ export default class Publisher { } } - async uploadToGithub( + public async deleteBatch(filePaths: string[]): Promise { + if (filePaths.length === 0) { + return true; + } + + try { + const userGardenConnection = new RepositoryConnection({ + gardenRepository: this.settings.githubRepo, + githubUserName: this.settings.githubUserName, + githubToken: this.settings.githubToken, + }); + + await userGardenConnection.deleteFiles(filePaths); + + return true; + } catch (error) { + console.error(error); + + return false; + } + } + + public async publishBatch(files: CompiledPublishFile[]): Promise { + const filesToPublish = files.filter((f) => + isPublishFrontmatterValid(f.frontmatter), + ); + + if (filesToPublish.length === 0) { + return true; + } + + try { + const userGardenConnection = new RepositoryConnection({ + gardenRepository: this.settings.githubRepo, + githubUserName: this.settings.githubUserName, + githubToken: this.settings.githubToken, + }); + + await userGardenConnection.updateFiles(filesToPublish); + + return true; + } catch (error) { + console.error(error); + + return false; + } + } + + private async uploadToGithub( path: string, content: string, remoteFileHash?: string, @@ -167,18 +215,18 @@ export default class Publisher { }); } - async uploadText(filePath: string, content: string, sha?: string) { + private async uploadText(filePath: string, content: string, sha?: string) { content = Base64.encode(content); const path = `${NOTE_PATH_BASE}${filePath}`; await this.uploadToGithub(path, content, sha); } - async uploadImage(filePath: string, content: string, sha?: string) { + private async uploadImage(filePath: string, content: string, sha?: string) { const path = `src/site${filePath}`; await this.uploadToGithub(path, content, sha); } - async uploadAssets(assets: Assets) { + private async uploadAssets(assets: Assets) { for (let idx = 0; idx < assets.images.length; idx++) { const image = assets.images[idx]; await this.uploadImage(image.path, image.content, image.remoteHash); diff --git a/src/repositoryConnection/RepositoryConnection.ts b/src/repositoryConnection/RepositoryConnection.ts index 14a9c343..96a975d6 100644 --- a/src/repositoryConnection/RepositoryConnection.ts +++ b/src/repositoryConnection/RepositoryConnection.ts @@ -1,9 +1,14 @@ import { Octokit } from "@octokit/core"; import Logger from "js-logger"; +import { CompiledPublishFile } from "src/publishFile/PublishFile"; const logger = Logger.get("repository-connection"); const oktokitLogger = Logger.get("octokit"); +//TODO: Move to global constants +const IMAGE_PATH_BASE = "src/site/img/user/"; +const NOTE_PATH_BASE = "src/site/notes/"; + interface IOctokitterInput { githubToken: string; githubUserName: string; @@ -157,10 +162,12 @@ export class RepositoryConnection { } } - async getLatestCommit(): Promise<{ sha: string } | undefined> { + async getLatestCommit(): Promise< + { sha: string; commit: { tree: { sha: string } } } | undefined + > { try { const latestCommit = await this.octokit.request( - "GET /repos/{owner}/{repo}/commits/HEAD", + `GET /repos/{owner}/{repo}/commits/HEAD?cacheBust=${Date.now()}`, this.getBasePayload(), ); @@ -194,6 +201,216 @@ export class RepositoryConnection { } } + async deleteFiles(filePaths: string[]) { + const latestCommit = await this.getLatestCommit(); + + if (!latestCommit) { + logger.error("Could not get latest commit"); + + return; + } + + const normalizePath = (path: string) => + path.startsWith("/") ? path.slice(1) : path; + + const filesToDelete = filePaths.map((path) => { + if (path.endsWith(".md")) { + return `${NOTE_PATH_BASE}${normalizePath(path)}`; + } + + return `${IMAGE_PATH_BASE}${normalizePath(path)}`; + }); + + const repoDataPromise = this.octokit.request( + "GET /repos/{owner}/{repo}", + { + ...this.getBasePayload(), + }, + ); + + const latestCommitSha = latestCommit.sha; + const baseTreeSha = latestCommit.commit.tree.sha; + + const baseTree = await this.octokit.request( + "GET /repos/{owner}/{repo}/git/trees/{tree_sha}?recursive=1", + { + ...this.getBasePayload(), + tree_sha: baseTreeSha, + }, + ); + + const newTreeEntries = baseTree.data.tree + .filter( + (item: { path: string }) => !filesToDelete.includes(item.path), + ) // Exclude files to delete + .map( + (item: { + path: string; + mode: string; + type: string; + sha: string; + }) => ({ + path: item.path, + mode: item.mode, + type: item.type, + sha: item.sha, + }), + ); + + const newTree = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/trees", + { + ...this.getBasePayload(), + base_tree: baseTreeSha, + tree: newTreeEntries, + }, + ); + + const commitMessage = "Deleted multiple files"; + + const newCommit = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/commits", + { + ...this.getBasePayload(), + message: commitMessage, + tree: newTree.data.sha, + parents: [latestCommitSha], + }, + ); + + const defaultBranch = (await repoDataPromise).data.default_branch; + + await this.octokit.request( + "PATCH /repos/{owner}/{repo}/git/refs/heads/{branch}", + { + ...this.getBasePayload(), + branch: defaultBranch, + sha: newCommit.data.sha, + }, + ); + } + + async updateFiles(files: CompiledPublishFile[]) { + const latestCommit = await this.getLatestCommit(); + + if (!latestCommit) { + logger.error("Could not get latest commit"); + + return; + } + + const repoDataPromise = this.octokit.request( + "GET /repos/{owner}/{repo}", + { + ...this.getBasePayload(), + }, + ); + + const latestCommitSha = latestCommit.sha; + const baseTreeSha = latestCommit.commit.tree.sha; + + const normalizePath = (path: string) => + path.startsWith("/") ? path.slice(1) : path; + + const treePromises = files.map(async (file) => { + const [text, _] = file.compiledFile; + + try { + const blob = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/blobs", + { + ...this.getBasePayload(), + content: text, + encoding: "utf-8", + }, + ); + + return { + path: `${NOTE_PATH_BASE}${normalizePath(file.getPath())}`, + mode: "100644", + type: "blob", + sha: blob.data.sha, + }; + } catch (error) { + logger.error(error); + } + }); + + const treeAssetPromises = files + .flatMap((x) => x.compiledFile[1].images) + .map(async (asset) => { + try { + const blob = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/blobs", + { + ...this.getBasePayload(), + content: asset.content, + encoding: "base64", + }, + ); + + return { + path: `${IMAGE_PATH_BASE}${normalizePath(asset.path)}`, + mode: "100644", + type: "blob", + sha: blob.data.sha, + }; + } catch (error) { + logger.error(error); + } + }); + treePromises.push(...treeAssetPromises); + + const treeList = await Promise.all(treePromises); + + //Filter away undefined values + const tree = treeList.filter((x) => x !== undefined) as { + path?: string | undefined; + mode?: + | "100644" + | "100755" + | "040000" + | "160000" + | "120000" + | undefined; + type?: "tree" | "blob" | "commit" | undefined; + sha?: string | null | undefined; + content?: string | undefined; + }[]; + + const newTree = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/trees", + { + ...this.getBasePayload(), + base_tree: baseTreeSha, + tree, + }, + ); + + const commitMessage = "Published multiple files"; + + const newCommit = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/commits", + { + ...this.getBasePayload(), + message: commitMessage, + tree: newTree.data.sha, + parents: [latestCommitSha], + }, + ); + + const defaultBranch = (await repoDataPromise).data.default_branch; + + await this.octokit.request( + "PATCH /repos/{owner}/{repo}/git/refs/heads/{branch}", + { + ...this.getBasePayload(), + branch: defaultBranch, + sha: newCommit.data.sha, + }, + ); + } + async getRepositoryInfo() { const repoInfo = await this.octokit .request("GET /repos/{owner}/{repo}", { diff --git a/src/test/snapshot/snapshot.md b/src/test/snapshot/snapshot.md index d25546dc..feb6017a 100644 --- a/src/test/snapshot/snapshot.md +++ b/src/test/snapshot/snapshot.md @@ -730,7 +730,7 @@ P Plugins/PD Dataview/PD4 DataviewJs queries.md

Header 2

PD4 DataviewJs queries

-
name6link
006 Custom title006 Custom title
005 Custom filters005 Custom filters
007 Custom permalink007 Custom permalink
011 Custom updatedAt011 Custom updatedAt
013 Custom path013 Custom path
014 Customer path and permalink014 Customer path and permalink
+
name6link
005 Custom filters005 Custom filters
006 Custom title006 Custom title
007 Custom permalink007 Custom permalink
011 Custom updatedAt011 Custom updatedAt
013 Custom path013 Custom path
014 Customer path and permalink014 Customer path and permalink
/img/user/A Assets/travolta.png diff --git a/src/views/PublicationCenter/PublicationCenter.svelte b/src/views/PublicationCenter/PublicationCenter.svelte index b34fe6a7..a791729c 100644 --- a/src/views/PublicationCenter/PublicationCenter.svelte +++ b/src/views/PublicationCenter/PublicationCenter.svelte @@ -184,45 +184,20 @@ showPublishingView = true; - for (const note of changedToPublish.concat(unpublishedToPublish)) { - processingPaths.push(note.getPath()); - let isPublished = await publisher.publish(note); + const allNotesToPublish = unpublishedToPublish.concat(changedToPublish); - processingPaths = processingPaths.filter( - (path) => path !== note.getPath(), - ); + processingPaths = [...allNotesToPublish.map((note) => note.getPath())]; + await publisher.publishBatch(allNotesToPublish); - if (isPublished) { - publishedPaths = [...publishedPaths, note.getPath()]; - } else { - failedPublish = [...failedPublish, note.getPath()]; - } - } + publishedPaths = [...processingPaths]; - for (const path of [...notesToDelete, ...imagesToDelete]) { - processingPaths.push(path); - const isNote = path.endsWith(".md"); - let isDeleted: boolean; + const allNotesToDelete = [...notesToDelete, ...imagesToDelete]; + processingPaths = [...allNotesToDelete]; - if (isNote) { - const sha = publishStatus.deletedNotePaths.find( - (p) => p.path === path, - )?.sha; - - isDeleted = await publisher.deleteNote(path, sha); - } else { - // TODO: remove with sha - isDeleted = await publisher.deleteImage(path); - } - - processingPaths = processingPaths.filter((p) => p !== path); - - if (isDeleted) { - publishedPaths = [...publishedPaths, path]; - } else { - failedPublish = [...failedPublish, path]; - } - } + // need to wait for about 1 second to allow github to process the publish request + await publisher.deleteBatch(allNotesToDelete); + publishedPaths = [...publishedPaths, ...processingPaths]; + processingPaths = []; }; const emptyNode: TreeNode = { diff --git a/src/views/PublishStatusBar.ts b/src/views/PublishStatusBar.ts index ae991773..07ac8ec2 100644 --- a/src/views/PublishStatusBar.ts +++ b/src/views/PublishStatusBar.ts @@ -16,6 +16,11 @@ export class PublishStatusBar { }); } + incrementMultiple(increments: number) { + this.counter += increments; + + this.status.innerText = `⌛Publishing files: ${this.counter}/${this.numberOfNotesToPublish}`; + } increment() { this.status.innerText = `⌛Publishing files: ${++this.counter}/${ this.numberOfNotesToPublish diff --git a/versions.json b/versions.json index 90197928..61fb3696 100644 --- a/versions.json +++ b/versions.json @@ -1,4 +1,5 @@ { + "2.57.0": "0.12.0", "2.56.2": "0.12.0", "2.56.1": "0.12.0", "2.56.0": "0.12.0",