feat(website): Excel templates (#3543)

docs/src/content/docs/for-users/submit-sequences.md

@@ -9,8 +9,7 @@ Before you begin this process, you should ensure your data is in the correct for
 Loculus expects:
 
 -   Sequence data in `fasta` format with a unique submissionID per sequence.
--   Metadata in `tsv` format for each sequence. If you upload through the Website, you can also use Excel files (`xls` or `xlsx` format). If you need help formatting metadata, there is a metadata template for each organism on the submission page.
-You can also map columns in your file to the expected upload column names by clicking the 'Add column mapping' button.
+-   Metadata for each sequence. If you upload through the API, only `tsv` is supported. If you upload through the Website, you can also use Excel files (`xlsx` format). If you need help formatting metadata, there is a metadata template for each organism on the submission page in each of the supported formats. You can also map columns in your file to the expected upload column names by clicking the 'Add column mapping' button.
 
 ![Metadata template.](../../../assets/MetadataTemplate.png)
 

website/src/components/Submission/DataUploadForm.tsx

@@ -236,11 +236,21 @@ const InnerDataUploadForm = ({
                             <a href='/docs/concepts/metadataformat' className='text-primary-700 opacity-90'>
                                 metadata format
                             </a>
-                            . You can download{' '}
-                            <a href={routes.metadataTemplate(organism, action)} className='text-primary-700 opacity-90'>
-                                a template
+                            . You can download a{' '}
+                            <a
+                                href={routes.metadataTemplate(organism, action, 'tsv')}
+                                className='text-primary-700 opacity-90'
+                            >
+                                TSV
+                            </a>
+                            {' or '}
+                            <a
+                                href={routes.metadataTemplate(organism, action, 'xlsx')}
+                                className='text-primary-700 opacity-90'
+                            >
+                                XLSX
                             </a>{' '}
-                            for the TSV metadata file with column headings.
+                            template with column headings for the metadata file.
                         </p>
 
                         {isMultiSegmented && (

website/src/pages/[organism]/submission/template/index.ts

@@ -1,9 +1,17 @@
 import type { APIRoute } from 'astro';
+import * as XLSX from 'xlsx';
 
 import { cleanOrganism } from '../../../../components/Navigation/cleanOrganism';
 import type { UploadAction } from '../../../../components/Submission/DataUploadForm.tsx';
 import { getMetadataTemplateFields } from '../../../../config';
 
+export type TemplateFileType = 'tsv' | 'xlsx';
+const VALID_FILE_TYPES = ['tsv', 'xlsx'];
+const CONTENT_TYPES = new Map<TemplateFileType, string>([
+    ['tsv', 'text/tab-separated-values'],
+    ['xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
+]);
+
 /** The TSV template file that users can download from the submission page. */
 export const GET: APIRoute = ({ params, request }) => {
     const rawOrganism = params.organism!;
@@ -14,18 +22,42 @@ export const GET: APIRoute = ({ params, request }) => {
         });
     }
 
-    const action: UploadAction = new URL(request.url).searchParams.get('format') === 'revise' ? 'revise' : 'submit';
-    const fieldNames = getMetadataTemplateFields(organism.key, action);
-    const tsvTemplate = [...fieldNames.keys()].join('\t') + '\n';
+    const searchParams = new URL(request.url).searchParams;
+    const action: UploadAction = searchParams.get('format') === 'revise' ? 'revise' : 'submit';
+    const fileTypeStr = searchParams.get('fileType')?.toLowerCase() ?? '';
+    const fileType: TemplateFileType = VALID_FILE_TYPES.includes(fileTypeStr)
+        ? (fileTypeStr as TemplateFileType)
+        : 'tsv';
+
+    const filename = `${organism.displayName.replaceAll(' ', '_')}_metadata_${action === 'revise' ? 'revision_' : ''}template.${fileType}`;
 
+    /* eslint-disable @typescript-eslint/naming-convention */
     const headers: Record<string, string> = {
-        'Content-Type': 'text/tsv', // eslint-disable-line @typescript-eslint/naming-convention
+        'Content-Type': CONTENT_TYPES.get(fileType)!,
+        'Content-Disposition': `attachment; filename="${filename}"`,
     };
+    /* eslint-enable @typescript-eslint/naming-convention */
 
-    const filename = `${organism.displayName.replaceAll(' ', '_')}_metadata_${action === 'revise' ? 'revision_' : ''}template.tsv`;
-    headers['Content-Disposition'] = `attachment; filename="${filename}"`;
+    const columnNames = Array.from(getMetadataTemplateFields(organism.key, action).keys());
 
-    return new Response(tsvTemplate, {
+    const fileBuffer = createTemplateFile(fileType, columnNames);
+
+    return new Response(fileBuffer, {
         headers,
     });
 };
+
+function createTemplateFile(fileType: TemplateFileType, columnNames: string[]): Uint8Array {
+    if (fileType === 'tsv') {
+        const content = columnNames.join('\t') + '\n';
+        return new TextEncoder().encode(content);
+    }
+
+    const worksheetData = [columnNames]; // Add headers as the first row
+    const worksheet = XLSX.utils.aoa_to_sheet(worksheetData);
+
+    const workbook = XLSX.utils.book_new();
+    XLSX.utils.book_append_sheet(workbook, worksheet, 'Template');
+
+    return XLSX.write(workbook, { type: 'buffer', bookType: fileType }) as Uint8Array;
+}

website/src/routes/routes.ts

@@ -1,5 +1,6 @@
 import { SubmissionRouteUtils } from './SubmissionRoute.ts';
 import type { UploadAction } from '../components/Submission/DataUploadForm.tsx';
+import type { TemplateFileType } from '../pages/[organism]/submission/template/index.ts';
 import { type AccessionVersion } from '../types/backend.ts';
 import { FileType } from '../types/lapis.ts';
 import { getAccessionVersionString } from '../utils/extractAccessionVersion.ts';
@@ -12,8 +13,8 @@ export const routes = {
     apiDocumentationPage: () => '/api-documentation',
     organismStartPage: (organism: string) => `/${organism}`,
     searchPage: (organism: string) => withOrganism(organism, `/search`),
-    metadataTemplate: (organism: string, format: UploadAction) =>
-        withOrganism(organism, `/submission/template?format=${format}`),
+    metadataTemplate: (organism: string, format: UploadAction, fileType: TemplateFileType) =>
+        withOrganism(organism, `/submission/template?format=${format}&fileType=${fileType}`),
     metadataOverview: (organism: string) => withOrganism(organism, `/metadata-overview`),
 
     mySequencesPage: (organism: string, groupId: number) =>

website/tests/pages/revise/revise.page.ts

@@ -20,9 +20,17 @@ export class RevisePage {
         await this.page.getByRole('button', { name: 'Submit' }).click();
     }
 
-    public async downloadMetadataTemplate() {
+    public async downloadTsvMetadataTemplate() {
+        return this.downloadMetadataTemplate('TSV');
+    }
+
+    public async downloadXlsxMetadataTemplate() {
+        return this.downloadMetadataTemplate('XLSX');
+    }
+
+    private async downloadMetadataTemplate(format: 'TSV' | 'XLSX') {
         const downloadPromise = this.page.waitForEvent('download');
-        await this.page.getByText('a template', { exact: true }).click();
+        await this.page.getByText(format, { exact: true }).click();
         return downloadPromise;
     }
 

website/tests/pages/submission/submit.page.ts

@@ -57,9 +57,17 @@ export class SubmitPage {
         await this.page.click(restrictedSelector);
     }
 
-    public async downloadMetadataTemplate() {
+    public async downloadTsvMetadataTemplate() {
+        return this.downloadMetadataTemplate('TSV');
+    }
+
+    public async downloadXlsxMetadataTemplate() {
+        return this.downloadMetadataTemplate('XLSX');
+    }
+
+    private async downloadMetadataTemplate(format: 'TSV' | 'XLSX') {
         const downloadPromise = this.page.waitForEvent('download');
-        await this.page.getByText('a template', { exact: true }).click();
+        await this.page.getByText(format, { exact: true }).click();
         return downloadPromise;
     }
 }

website/tests/pages/submission/template.spec.ts

@@ -1,4 +1,5 @@
 import type { Download } from '@playwright/test';
+import * as XLSX from 'xlsx';
 
 import { expect, test } from '../../e2e.fixture.ts';
 
@@ -13,11 +14,18 @@ test.describe('The submit page', () => {
         const { groupId } = await loginAsTestUser();
         await submitPage.goto(groupId);
 
-        const download = await submitPage.downloadMetadataTemplate();
+        let download = await submitPage.downloadTsvMetadataTemplate();
+
+        const expectedHeaders = ['submissionId', 'country', 'date'];
 
         expect(download.suggestedFilename()).toBe('Test_Dummy_Organism_metadata_template.tsv');
-        const content = await getDownloadedContent(download);
+        const content = await getDownloadedContentAsString(download);
         expect(content).toStrictEqual('submissionId\tcountry\tdate\n');
+
+        download = await submitPage.downloadXlsxMetadataTemplate();
+        expect(download.suggestedFilename()).toBe('Test_Dummy_Organism_metadata_template.xlsx');
+        const workbook = await getDownloadedContentAsExcel(download);
+        expectHeaders(workbook, expectedHeaders);
     });
 
     test('should download the metadata file template for revision', async ({
@@ -30,22 +38,54 @@ test.describe('The submit page', () => {
         const { groupId } = await loginAsTestUser();
         await revisePage.goto(groupId);
 
-        const download = await revisePage.downloadMetadataTemplate();
+        let download = await revisePage.downloadTsvMetadataTemplate();
+
+        const expectedHeaders = ['accession', 'submissionId', 'country', 'date'];
 
         expect(download.suggestedFilename()).toBe('Test_Dummy_Organism_metadata_revision_template.tsv');
-        const content = await getDownloadedContent(download);
+        const content = await getDownloadedContentAsString(download);
         expect(content).toStrictEqual('accession\tsubmissionId\tcountry\tdate\n');
+
+        download = await revisePage.downloadXlsxMetadataTemplate();
+        expect(download.suggestedFilename()).toBe('Test_Dummy_Organism_metadata_revision_template.xlsx');
+        const workbook = await getDownloadedContentAsExcel(download);
+        expectHeaders(workbook, expectedHeaders);
     });
 
-    async function getDownloadedContent(download: Download) {
+    async function getDownloadedContent(download: Download): Promise<ArrayBuffer> {
         const readable = await download.createReadStream();
-        return new Promise((resolve) => {
-            let data = '';
-            readable.on('data', (chunk) => (data += chunk as string));
-            readable.on('end', () => resolve(data));
+        return new Promise((resolve, reject) => {
+            const chunks: Buffer[] = [];
+            readable.on('data', (chunk) => chunks.push(chunk as Buffer));
+            readable.on('end', () => {
+                const buffer = Buffer.concat(chunks);
+                resolve(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
+            });
+            readable.on('error', reject);
         });
     }
 
+    async function getDownloadedContentAsString(download: Download): Promise<string> {
+        const arrayBuffer = await getDownloadedContent(download);
+        return new TextDecoder().decode(arrayBuffer);
+    }
+
+    async function getDownloadedContentAsExcel(download: Download): Promise<XLSX.WorkBook> {
+        const arrayBuffer = await getDownloadedContent(download);
+        return XLSX.read(arrayBuffer);
+    }
+
+    function expectHeaders(workBook: XLSX.WorkBook, headers: string[]) {
+        expect(workBook.SheetNames.length).toBe(1);
+        const sheet = workBook.Sheets[workBook.SheetNames[0]];
+
+        const arrayOfArrays = XLSX.utils.sheet_to_json(sheet, { header: 1 });
+        expect(arrayOfArrays.length).toBeGreaterThan(0);
+
+        const sheetHeaders = arrayOfArrays[0];
+        expect(sheetHeaders).toEqual(headers);
+    }
+
     function skipDownloadTestInWebkit(browserName: 'chromium' | 'firefox' | 'webkit') {
         test.skip(
             browserName === 'webkit',