merged master & resolved conflicts

- rebuilt docs
- removed moonbeam specific code from xcm decoder

CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [17.3.3](https://github.com/paritytech/substrate-api-sidecar/compare/v17.3.2..v17.3.3) (2023-12-21)
+
+**NOTE** This release focuses on improving the performance of the tool resolving a regression where `blocks` were overwhelmed with transactions. The `noFees` query parameter focuses on removing fee info for the blocks if the user does not intend on needing fees. For more general cases where fees are necessary we have increased the performance of querying `/blocks` while also calculating fees. This was done with 2 cases: ensuring `transactionPaidFee`, and `ExtrinsicSuccess` or `ExtrinsicFailure` info is used to its fullest so we can avoid making any additional rpc calls, as well as ensuring the extrinsic's are called concurrently.
+### Perf
+
+- perf: transactionPaidFee event optimization ([#1367](https://github.com/paritytech/substrate-api-sidecar/pull/1367)) ([2883249](https://github.com/paritytech/substrate-api-sidecar/commit/288324918b7445fce8be6c4606c41058c66cdf69)) 
+- perf: add concurrency to fee calls ([#1368](https://github.com/paritytech/substrate-api-sidecar/pull/1368)) ([0980d1e](https://github.com/paritytech/substrate-api-sidecar/commit/0980d1e0766f1e416883c8e68ebcf04ca8ecd2fc))
+
+### Fix
+
+- fix: add finalizedKey query param to /blocks/{blockId} ([#1362](https://github.com/paritytech/substrate-api-sidecar/pull/1362)) ([ecd1518](https://github.com/paritytech/substrate-api-sidecar/commit/ecd1518a1e7f6b4465e90cd7ff8f6fd7a115b88b))
+- fix: added query to calc fees ([#1366](https://github.com/paritytech/substrate-api-sidecar/pull/1366)) ([203a257](https://github.com/paritytech/substrate-api-sidecar/commit/203a257bea3c150ac09d7b7fa956d68e524969bc))
+    NOTE: this added the noFees={bool} query param to specify whether to retrieve or not the fees information of the block for the `/blocks/*` endpoint
+
+### Test
+
+-  test(e2e): replace tests pointing to deprecated pallets ([#1363](https://github.com/paritytech/substrate-api-sidecar/pull/1363)) ([2a38b2e](https://github.com/paritytech/substrate-api-sidecar/commit/2a38b2e652bf621bf6cc450ce5fdacf5656d9c8c))
+
+## Compatibility
+
+Tested against:
+- Polkadot v10400
+- Kusama v10400
+- Westend v10400
+
 ## [17.3.2](https://github.com/paritytech/substrate-api-sidecar/compare/v17.3.1..v17.3.2) (2023-11-26)
 
 ### Fix

docs/src/openapi-v1.yaml

@@ -8,7 +8,7 @@ info:
   license:
     name: GPL-3.0-or-later
     url: https://github.com/paritytech/substrate-api-sidecar/blob/master/LICENSE
-  version: 17.3.2
+  version: 17.3.3
 servers:
 - url: https://polkadot-public-sidecar.parity-chains.parity.io/
   description: Polkadot Parity public sidecar
@@ -576,6 +576,13 @@ paths:
         schema:
           type: boolean
           default: false
+      - name: noFees
+        in: query
+        description: When set to `true`, the fee won't be calculated for the extrinsics.
+        required: false
+        schema:
+          type: boolean
+          default: false
       responses:
         "200":
           description: successful operation
@@ -621,6 +628,13 @@ paths:
         schema:
           type: boolean
           default: false
+      - name: noFees
+        in: query
+        description: When set to `true`, the fee won't be calculated for the extrinsics.
+        required: false
+        schema:
+          type: boolean
+          default: false
       - name: finalizedKey
         in: query
         description: When set to false, this will override the chain-config, and omit the
@@ -772,6 +786,13 @@ paths:
         schema:
           type: boolean
           default: false
+      - name: noFees
+        in: query
+        description: When set to `true`, the fee won't be calculated for the extrinsics.
+        required: false
+        schema:
+          type: boolean
+          default: false
       responses:
         "200":
           description: successful operation

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "17.3.2",
+  "version": "17.3.3",
   "name": "@substrate/api-sidecar",
   "description": "REST service that makes it easy to interact with blockchain nodes built using Substrate's FRAME framework.",
   "homepage": "https://github.com/paritytech/substrate-api-sidecar#readme",

src/controllers/blocks/BlocksController.ts

@@ -116,7 +116,7 @@ export default class BlocksController extends AbstractController<BlocksService>
 	 * @param _req Express Request
 	 * @param res Express Response
 	 */
-	private getLatestBlock: RequestHandler = async ({ query: { eventDocs, extrinsicDocs, finalized } }, res) => {
+	private getLatestBlock: RequestHandler = async ({ query: { eventDocs, extrinsicDocs, finalized, noFees } }, res) => {
 		const eventDocsArg = eventDocs === 'true';
 		const extrinsicDocsArg = extrinsicDocs === 'true';
 
@@ -138,13 +138,15 @@ export default class BlocksController extends AbstractController<BlocksService>
 			queryFinalizedHead = false;
 			hash = await this.api.rpc.chain.getFinalizedHead();
 		}
+		const noFeesArg = noFees === 'true';
 
 		const options = {
 			eventDocs: eventDocsArg,
 			extrinsicDocs: extrinsicDocsArg,
 			checkFinalized: false,
 			queryFinalizedHead,
 			omitFinalizedTag,
+			noFees: noFeesArg,
 		};
 
 		const historicApi = await this.api.at(hash);
@@ -159,7 +161,7 @@ export default class BlocksController extends AbstractController<BlocksService>
 	 * @param res Express Response
 	 */
 	private getBlockById: RequestHandler<INumberParam> = async (
-		{ params: { number }, query: { eventDocs, extrinsicDocs, finalizedKey, decodedXcmMsgs, paraId } },
+		{ params: { number }, query: { eventDocs, extrinsicDocs, noFees, finalizedKey, decodedXcmMsgs, paraId } },
 		res,
 	): Promise<void> => {
 		const checkFinalized = isHex(number);
@@ -172,6 +174,7 @@ export default class BlocksController extends AbstractController<BlocksService>
 		const finalizeOverride = finalizedKey === 'false';
 
 		const queryFinalizedHead = !this.options.finalizes ? false : true;
+		const noFeesArg = noFees === 'true';
 		let omitFinalizedTag = !this.options.finalizes ? true : false;
 
 		if (finalizeOverride) {
@@ -184,6 +187,7 @@ export default class BlocksController extends AbstractController<BlocksService>
 			checkFinalized,
 			queryFinalizedHead,
 			omitFinalizedTag,
+			noFees: noFeesArg,
 		};
 
 		// HistoricApi to fetch any historic information that doesnt include the current runtime
@@ -229,7 +233,7 @@ export default class BlocksController extends AbstractController<BlocksService>
 	 * @param res Express Response
 	 */
 	private getBlocks: RequestHandler<unknown, unknown, unknown, IRangeQueryParam> = async (
-		{ query: { range, eventDocs, extrinsicDocs } },
+		{ query: { range, eventDocs, extrinsicDocs, noFees } },
 		res,
 	): Promise<void> => {
 		if (!range) throw new BadRequest('range query parameter must be inputted.');
@@ -241,12 +245,14 @@ export default class BlocksController extends AbstractController<BlocksService>
 		const extrinsicDocsArg = extrinsicDocs === 'true';
 		const queryFinalizedHead = !this.options.finalizes ? false : true;
 		const omitFinalizedTag = !this.options.finalizes ? true : false;
+		const noFeesArg = noFees === 'true';
 		const options = {
 			eventDocs: eventDocsArg,
 			extrinsicDocs: extrinsicDocsArg,
 			checkFinalized: false,
 			queryFinalizedHead,
 			omitFinalizedTag,
+			noFees: noFeesArg,
 		};
 
 		const pQueue = new PromiseQueue(4);

src/controllers/blocks/BlocksExtrinsicsController.ts

@@ -42,20 +42,22 @@ export default class BlocksExtrinsicsController extends AbstractController<Block
 	 * @param res Express Response
 	 */
 	private getExtrinsicByTimepoint: RequestHandler<INumberParam> = async (
-		{ params: { blockId, extrinsicIndex }, query: { eventDocs, extrinsicDocs } },
+		{ params: { blockId, extrinsicIndex }, query: { eventDocs, extrinsicDocs, noFees } },
 		res,
 	): Promise<void> => {
 		const hash = await this.getHashForBlock(blockId);
 
 		const eventDocsArg = eventDocs === 'true';
 		const extrinsicDocsArg = extrinsicDocs === 'true';
+		const noFeesArg = noFees === 'true';
 
 		const options = {
 			eventDocs: eventDocsArg,
 			extrinsicDocs: extrinsicDocsArg,
 			checkFinalized: true,
 			queryFinalizedHead: false,
 			omitFinalizedTag: true,
+			noFees: noFeesArg,
 		};
 
 		const historicApi = await this.api.at(hash);

src/services/blocks/BlocksService.spec.ts

@@ -122,6 +122,7 @@ describe('BlocksService', () => {
 				checkFinalized: false,
 				queryFinalizedHead: false,
 				omitFinalizedTag: false,
+				noFees: false,
 			};
 
 			expect(sanitizeNumbers(await blocksService.fetchBlock(blockHash789629, mockHistoricApi, options))).toMatchObject(
@@ -146,6 +147,7 @@ describe('BlocksService', () => {
 				checkFinalized: false,
 				queryFinalizedHead: false,
 				omitFinalizedTag: false,
+				noFees: false,
 			};
 			const tempGetBlock = mockApi.rpc.chain.getBlock;
 			mockApi.rpc.chain.getBlock = (() =>
@@ -170,6 +172,7 @@ describe('BlocksService', () => {
 				checkFinalized: false,
 				queryFinalizedHead: false,
 				omitFinalizedTag: true,
+				noFees: false,
 			};
 
 			const block = await blocksService.fetchBlock(blockHash789629, mockHistoricApi, options);
@@ -357,6 +360,7 @@ describe('BlocksService', () => {
 			checkFinalized: false,
 			queryFinalizedHead: false,
 			omitFinalizedTag: false,
+			noFees: false,
 		};
 
 		it('Returns the correct extrinisics object for block 789629', async () => {
@@ -442,6 +446,7 @@ describe('BlocksService', () => {
 			checkFinalized: false,
 			queryFinalizedHead: false,
 			omitFinalizedTag: false,
+			noFees: false,
 		};
 
 		it('Should correctly store the most recent queried blocks', async () => {