From c58fa1b0c33e8ee829f20cc93664dc69bc8d0780 Mon Sep 17 00:00:00 2001 From: Irfan Date: Thu, 27 Jun 2024 21:03:36 +0500 Subject: [PATCH 01/31] docs: documents ServiceUnavailableException #514 --- config/swagger-lume.php | 1 + storage/api-docs/api-docs.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/swagger-lume.php b/config/swagger-lume.php index f9182e3c..d5ffb8ac 100644 --- a/config/swagger-lume.php +++ b/config/swagger-lume.php @@ -260,6 +260,7 @@ | `BadRequestException` | `405 - Method Not Allowed` | Requested Method is not supported for resource. Only `GET` requests are allowed | | `RateLimitException` | `429 - Too Many Request` | You are being rate limited by Jikan or MyAnimeList is rate-limiting our servers (specified in the error response) | | `UpstreamException`,`ParserException`,etc. | `500 - Internal Server Error` | Something didn't work. Try again later. If you see an error response with a `report_url` URL, please click on it to open an auto-generated GitHub issue | + | `ServiceUnavailableException` | `503 - Service Unavailable` | In most cases this is intentionally done if the service is down for maintenance. | ## JSON Error Response diff --git a/storage/api-docs/api-docs.json b/storage/api-docs/api-docs.json index 8ab520ac..fb324e94 100644 --- a/storage/api-docs/api-docs.json +++ b/storage/api-docs/api-docs.json @@ -2,7 +2,7 @@ "openapi": "3.0.0", "info": { "title": "Jikan API", - "description": "[Jikan](https://jikan.moe) is an **Unofficial** MyAnimeList API.\nIt scrapes the website to satisfy the need for a complete API - which MyAnimeList lacks.\n\n# Information\n\nāš” Jikan is powered by its awesome backers - šŸ™ [Become a backer](https://www.patreon.com/jikan)\n\n## Rate Limiting\n\n| Duration | Requests |\n|----|----|\n| Daily | **Unlimited** |\n| Per Minute | 60 requests |\n| Per Second | 3 requests |\n\nNote: It's still possible to get rate limited from MyAnimeList.net instead.\n\n\n## JSON Notes\n- Any property (except arrays or objects) whose value does not exist or is undetermined, will be `null`.\n- Any array or object property whose value does not exist or is undetermined, will be empty.\n- Any `score` property whose value does not exist or is undetermined, will be `0`.\n- All dates and timestamps are returned in [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format and in UTC timezone\n\n## Caching\nBy **CACHING**, we refer to the data parsed from MyAnimeList which is stored temporarily on our servers to provide better API performance.\n\nAll requests are cached for **24 hours**.\n\nThe following response headers will detail cache information.\n\n| Header | Remarks |\n| ---- | ---- |\n| `Expires` | Cache expiry date |\n| `Last-Modified` | Cache set date |\n| `X-Request-Fingerprint` | Unique request fingerprint (only for cachable requests, not queries) |\n\n\nNote: `X-Request-Fingerprint` will only be available on single resource requests and their child endpoints. e.g `/anime/1`, `/anime/1/relations`.\nThey won't be available on pages which perform queries, like /anime, or /top/anime, etc.\n\n## Allowed HTTP(s) requests\n\n**Jikan REST API does not provide authenticated requests for MyAnimeList.** This means you can not use it to update your anime/manga list.\nOnly GET requests are supported which return READ-ONLY data.\n\n## HTTP Responses\n\nAll error responses are accompanied by a JSON Error response.\n\n| Exception | HTTP Status | Remarks |\n| ---- | ---- | ---- |\n| N/A | `200 - OK` | The request was successful |\n| N/A | `304 - Not Modified` | You have the latest data (Cache Validation response) |\n| `BadRequestException`,`ValidationException` | `400 - Bad Request` | You've made an invalid request. Recheck documentation |\n| `BadResponseException` | `404 - Not Found` | The resource was not found or MyAnimeList responded with a `404` |\n| `BadRequestException` | `405 - Method Not Allowed` | Requested Method is not supported for resource. Only `GET` requests are allowed |\n| `RateLimitException` | `429 - Too Many Request` | You are being rate limited by Jikan or MyAnimeList is rate-limiting our servers (specified in the error response) |\n| `UpstreamException`,`ParserException`,etc. | `500 - Internal Server Error` | Something didn't work. Try again later. If you see an error response with a `report_url` URL, please click on it to open an auto-generated GitHub issue |\n\n## JSON Error Response\n\n```json\n {\n \"status\": 500,\n \"type\": \"InternalException\",\n \"message\": \"Exception Message\",\n \"error\": \"Exception Trace\",\n \"report_url\": \"https://github.com...\"\n }\n```\n\n| Property | Remarks |\n| ---- | ---- |\n| `status` | Returned HTTP Status Code |\n| `type` | Thrown Exception |\n| `message` | Human-readable error message |\n| `error` | Error response and trace from the API |\n| `report_url` | Clicking this would redirect you to a generated GitHub issue |\n\n\n## Cache Validation\n\n- All requests return a `ETag` header which is an MD5 hash of the response\n- You can use this hash to verify if there's new or updated content by suppliying it as the value for the `If-None-Match` in your next request header\n- You will get a HTTP `304 - Not Modified` response if the content has not changed\n- If the content has changed, you'll get a HTTP `200 - OK` response with the updated JSON response\n\n![Cache Validation](https://i.imgur.com/925ozVn.png 'Cache Validation')\n\n## Disclaimer\n\n- Jikan is not affiliated with MyAnimeList.net.\n- Jikan is a free, open-source API. Please use it responsibly.\n\n----\n\nBy using the API, you are agreeing to Jikan's [terms of use](https://jikan.moe/terms) policy.\n\n[v3 Documentation](https://jikan.docs.apiary.io/) - [Wrappers/SDKs](https://github.com/jikan-me/jikan#wrappers) - [Report an issue](https://github.com/jikan-me/jikan-rest/issues/new) - [Host your own server](https://github.com/jikan-me/jikan-rest)", + "description": "[Jikan](https://jikan.moe) is an **Unofficial** MyAnimeList API.\nIt scrapes the website to satisfy the need for a complete API - which MyAnimeList lacks.\n\n# Information\n\nāš” Jikan is powered by its awesome backers - šŸ™ [Become a backer](https://www.patreon.com/jikan)\n\n## Rate Limiting\n\n| Duration | Requests |\n|----|----|\n| Daily | **Unlimited** |\n| Per Minute | 60 requests |\n| Per Second | 3 requests |\n\nNote: It's still possible to get rate limited from MyAnimeList.net instead.\n\n\n## JSON Notes\n- Any property (except arrays or objects) whose value does not exist or is undetermined, will be `null`.\n- Any array or object property whose value does not exist or is undetermined, will be empty.\n- Any `score` property whose value does not exist or is undetermined, will be `0`.\n- All dates and timestamps are returned in [ISO8601](https://en.wikipedia.org/wiki/ISO_8601) format and in UTC timezone\n\n## Caching\nBy **CACHING**, we refer to the data parsed from MyAnimeList which is stored temporarily on our servers to provide better API performance.\n\nAll requests are cached for **24 hours**.\n\nThe following response headers will detail cache information.\n\n| Header | Remarks |\n| ---- | ---- |\n| `Expires` | Cache expiry date |\n| `Last-Modified` | Cache set date |\n| `X-Request-Fingerprint` | Unique request fingerprint (only for cachable requests, not queries) |\n\n\nNote: `X-Request-Fingerprint` will only be available on single resource requests and their child endpoints. e.g `/anime/1`, `/anime/1/relations`.\nThey won't be available on pages which perform queries, like /anime, or /top/anime, etc.\n\n## Allowed HTTP(s) requests\n\n**Jikan REST API does not provide authenticated requests for MyAnimeList.** This means you can not use it to update your anime/manga list.\nOnly GET requests are supported which return READ-ONLY data.\n\n## HTTP Responses\n\nAll error responses are accompanied by a JSON Error response.\n\n| Exception | HTTP Status | Remarks |\n| ---- | ---- | ---- |\n| N/A | `200 - OK` | The request was successful |\n| N/A | `304 - Not Modified` | You have the latest data (Cache Validation response) |\n| `BadRequestException`,`ValidationException` | `400 - Bad Request` | You've made an invalid request. Recheck documentation |\n| `BadResponseException` | `404 - Not Found` | The resource was not found or MyAnimeList responded with a `404` |\n| `BadRequestException` | `405 - Method Not Allowed` | Requested Method is not supported for resource. Only `GET` requests are allowed |\n| `RateLimitException` | `429 - Too Many Request` | You are being rate limited by Jikan or MyAnimeList is rate-limiting our servers (specified in the error response) |\n| `UpstreamException`,`ParserException`,etc. | `500 - Internal Server Error` | Something didn't work. Try again later. If you see an error response with a `report_url` URL, please click on it to open an auto-generated GitHub issue |\n| `ServiceUnavailableException` | `503 - Service Unavailable` | In most cases this is intentionally done if the service is down for maintenance. |\n\n## JSON Error Response\n\n```json\n {\n \"status\": 500,\n \"type\": \"InternalException\",\n \"message\": \"Exception Message\",\n \"error\": \"Exception Trace\",\n \"report_url\": \"https://github.com...\"\n }\n```\n\n| Property | Remarks |\n| ---- | ---- |\n| `status` | Returned HTTP Status Code |\n| `type` | Thrown Exception |\n| `message` | Human-readable error message |\n| `error` | Error response and trace from the API |\n| `report_url` | Clicking this would redirect you to a generated GitHub issue |\n\n\n## Cache Validation\n\n- All requests return a `ETag` header which is an MD5 hash of the response\n- You can use this hash to verify if there's new or updated content by suppliying it as the value for the `If-None-Match` in your next request header\n- You will get a HTTP `304 - Not Modified` response if the content has not changed\n- If the content has changed, you'll get a HTTP `200 - OK` response with the updated JSON response\n\n![Cache Validation](https://i.imgur.com/925ozVn.png 'Cache Validation')\n\n## Disclaimer\n\n- Jikan is not affiliated with MyAnimeList.net.\n- Jikan is a free, open-source API. Please use it responsibly.\n\n----\n\nBy using the API, you are agreeing to Jikan's [terms of use](https://jikan.moe/terms) policy.\n\n[v3 Documentation](https://jikan.docs.apiary.io/) - [Wrappers/SDKs](https://github.com/jikan-me/jikan#wrappers) - [Report an issue](https://github.com/jikan-me/jikan-rest/issues/new) - [Host your own server](https://github.com/jikan-me/jikan-rest)", "termsOfService": "https://jikan.moe/terms", "contact": { "name": "API Support (Discord)", From 1de4a5a910820558c40170b95989780fe4fbe8de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 04:17:39 +0000 Subject: [PATCH 02/31] Bump docker/build-push-action from 5 to 6 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/container-image-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-image-release.yml b/.github/workflows/container-image-release.yml index bc73b124..ec4c2997 100644 --- a/.github/workflows/container-image-release.yml +++ b/.github/workflows/container-image-release.yml @@ -69,7 +69,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . platforms: ${{ matrix.platform }} From a6ec44dd24edbd17d1c02cd463e642a9fca2e5b4 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Tue, 2 Jul 2024 14:55:39 +0100 Subject: [PATCH 03/31] disable docker build summary --- .github/workflows/container-image-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container-image-release.yml b/.github/workflows/container-image-release.yml index ec4c2997..72e835b3 100644 --- a/.github/workflows/container-image-release.yml +++ b/.github/workflows/container-image-release.yml @@ -70,6 +70,8 @@ jobs: - name: Build and push by digest id: build uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_NO_SUMMARY: true with: context: . platforms: ${{ matrix.platform }} From 6d6c47d16e0f2271d9d503737f432424de22c8ed Mon Sep 17 00:00:00 2001 From: pushrbx Date: Thu, 11 Jul 2024 21:19:47 +0100 Subject: [PATCH 04/31] fixed api docs around the "spoilers" parameter --- app/Dto/Concerns/HasSpoilersParameter.php | 2 +- app/Http/Controllers/V4DB/AnimeController.php | 4 ++-- app/Http/Controllers/V4DB/MangaController.php | 4 ++-- app/Http/Controllers/V4DB/ReviewsController.php | 8 +++++--- storage/api-docs/api-docs.json | 12 ++++++------ 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/app/Dto/Concerns/HasSpoilersParameter.php b/app/Dto/Concerns/HasSpoilersParameter.php index b704e46d..7ba499cf 100644 --- a/app/Dto/Concerns/HasSpoilersParameter.php +++ b/app/Dto/Concerns/HasSpoilersParameter.php @@ -10,7 +10,7 @@ /** * @OA\Parameter( - * name="spoiler", + * name="spoilers", * in="query", * required=false, * description="Any reviews that are tagged as a spoiler. Spoiler reviews are not returned by default. e.g usage: `?spoiler=true`", diff --git a/app/Http/Controllers/V4DB/AnimeController.php b/app/Http/Controllers/V4DB/AnimeController.php index b6c26aff..161b2b66 100644 --- a/app/Http/Controllers/V4DB/AnimeController.php +++ b/app/Http/Controllers/V4DB/AnimeController.php @@ -22,7 +22,7 @@ use App\Dto\AnimeUserUpdatesLookupCommand; use App\Dto\AnimeVideosEpisodesLookupCommand; use App\Dto\AnimeVideosLookupCommand; -use Illuminate\Http\Request; +use OpenApi\Annotations as OA; class AnimeController extends Controller { @@ -669,7 +669,7 @@ public function userupdates(AnimeUserUpdatesLookupCommand $command) * * @OA\Parameter(ref="#/components/parameters/page"), * @OA\Parameter(ref="#/components/parameters/preliminary"), - * @OA\Parameter(ref="#/components/parameters/spoiler"), + * @OA\Parameter(ref="#/components/parameters/spoilers"), * * @OA\Response( * response="200", diff --git a/app/Http/Controllers/V4DB/MangaController.php b/app/Http/Controllers/V4DB/MangaController.php index e1794ebf..f863a2e2 100644 --- a/app/Http/Controllers/V4DB/MangaController.php +++ b/app/Http/Controllers/V4DB/MangaController.php @@ -15,7 +15,7 @@ use App\Dto\MangaReviewsLookupCommand; use App\Dto\MangaStatsLookupCommand; use App\Dto\MangaUserUpdatesLookupCommand; -use Illuminate\Http\Request; +use OpenApi\Annotations as OA; class MangaController extends Controller { @@ -387,7 +387,7 @@ public function userupdates(MangaUserUpdatesLookupCommand $command) * * @OA\Parameter(ref="#/components/parameters/page"), * @OA\Parameter(ref="#/components/parameters/preliminary"), - * @OA\Parameter(ref="#/components/parameters/spoiler"), + * @OA\Parameter(ref="#/components/parameters/spoilers"), * * @OA\Response( * response="200", diff --git a/app/Http/Controllers/V4DB/ReviewsController.php b/app/Http/Controllers/V4DB/ReviewsController.php index dcf8ac20..8dca6a95 100644 --- a/app/Http/Controllers/V4DB/ReviewsController.php +++ b/app/Http/Controllers/V4DB/ReviewsController.php @@ -4,18 +4,20 @@ use App\Dto\QueryAnimeReviewsCommand; use App\Dto\QueryMangaReviewsCommand; +use OpenApi\Annotations as OA; class ReviewsController extends Controller { /** - * @OA\Get( + * @OA\ + * Get( * path="/reviews/anime", * operationId="getRecentAnimeReviews", * tags={"reviews"}, * * @OA\Parameter(ref="#/components/parameters/page"), * @OA\Parameter(ref="#/components/parameters/preliminary"), - * @OA\Parameter(ref="#/components/parameters/spoiler"), + * @OA\Parameter(ref="#/components/parameters/spoilers"), * * * @OA\Response( @@ -70,7 +72,7 @@ public function anime(QueryAnimeReviewsCommand $command) * * @OA\Parameter(ref="#/components/parameters/page"), * @OA\Parameter(ref="#/components/parameters/preliminary"), - * @OA\Parameter(ref="#/components/parameters/spoiler"), + * @OA\Parameter(ref="#/components/parameters/spoilers"), * * @OA\Response( * response="200", diff --git a/storage/api-docs/api-docs.json b/storage/api-docs/api-docs.json index fb324e94..4779da6a 100644 --- a/storage/api-docs/api-docs.json +++ b/storage/api-docs/api-docs.json @@ -587,7 +587,7 @@ "$ref": "#/components/parameters/preliminary" }, { - "$ref": "#/components/parameters/spoiler" + "$ref": "#/components/parameters/spoilers" } ], "responses": { @@ -1605,7 +1605,7 @@ "$ref": "#/components/parameters/preliminary" }, { - "$ref": "#/components/parameters/spoiler" + "$ref": "#/components/parameters/spoilers" } ], "responses": { @@ -2226,7 +2226,7 @@ "$ref": "#/components/parameters/preliminary" }, { - "$ref": "#/components/parameters/spoiler" + "$ref": "#/components/parameters/spoilers" } ], "responses": { @@ -2258,7 +2258,7 @@ "$ref": "#/components/parameters/preliminary" }, { - "$ref": "#/components/parameters/spoiler" + "$ref": "#/components/parameters/spoilers" } ], "responses": { @@ -9086,8 +9086,8 @@ "type": "boolean" } }, - "spoiler": { - "name": "spoiler", + "spoilers": { + "name": "spoilers", "in": "query", "description": "Any reviews that are tagged as a spoiler. Spoiler reviews are not returned by default. e.g usage: `?spoiler=true`", "required": false, From 6e4ff113e360c4c62ba80ec08e8c0bd94eb27fce Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 26 Jul 2024 13:52:24 +0100 Subject: [PATCH 05/31] fixed enum cast for top reviews endpoint #548 --- app/Dto/QueryTopReviewsCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Dto/QueryTopReviewsCommand.php b/app/Dto/QueryTopReviewsCommand.php index b9ede8ef..45ca6ddd 100644 --- a/app/Dto/QueryTopReviewsCommand.php +++ b/app/Dto/QueryTopReviewsCommand.php @@ -23,6 +23,6 @@ final class QueryTopReviewsCommand extends QueryTopItemsCommand implements DataR { use HasRequestFingerprint, HasPreliminaryParameter, HasSpoilersParameter, PreparesData; - #[WithCast(EnumCast::class, TopAnimeFilterEnum::class), EnumValidation(TopReviewsTypeEnum::class)] + #[WithCast(EnumCast::class, TopReviewsTypeEnum::class), EnumValidation(TopReviewsTypeEnum::class)] public TopReviewsTypeEnum|Optional $type; } From a9f02dad8b9152e2fec59c8306e66ee450af2c3d Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 26 Jul 2024 14:14:13 +0100 Subject: [PATCH 06/31] added tests for top/reviews endpoint --- tests/Integration/TopControllerTest.php | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/Integration/TopControllerTest.php b/tests/Integration/TopControllerTest.php index e85ca9ff..f445066d 100644 --- a/tests/Integration/TopControllerTest.php +++ b/tests/Integration/TopControllerTest.php @@ -14,6 +14,15 @@ class TopControllerTest extends TestCase use SyntheticMongoDbTransaction; use ScoutFlush; + public function topReviewTypeParametersProvider(): array + { + return [ + "empty query string" => [[]], + "query string = `?type=anime`" => [["type" => "anime"]], + "query string = `?type=manga`" => [["type" => "manga"]], + ]; + } + public function testTopAnime() { Anime::factory(3)->state(new Sequence( @@ -290,4 +299,15 @@ public function test404() $this->get('/v4/top/anime/999') ->seeStatusCode(404); } + + /** + * @dataProvider topReviewTypeParametersProvider + * @param $params + * @return void + */ + public function testTopReviews($params) + { + $this->getJsonResponse($params,"/v4/top/reviews"); + $this->seeStatusCode(200); + } } From 475633457aa85c157e3728cce752af274bf8d9e0 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 26 Jul 2024 14:16:42 +0100 Subject: [PATCH 07/31] cleanup --- app/Dto/QueryTopReviewsCommand.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Dto/QueryTopReviewsCommand.php b/app/Dto/QueryTopReviewsCommand.php index 45ca6ddd..322b66f1 100644 --- a/app/Dto/QueryTopReviewsCommand.php +++ b/app/Dto/QueryTopReviewsCommand.php @@ -2,14 +2,12 @@ namespace App\Dto; -use App\Casts\ContextualBooleanCast; use App\Casts\EnumCast; use App\Concerns\HasRequestFingerprint; use App\Contracts\DataRequest; use App\Dto\Concerns\HasPreliminaryParameter; use App\Dto\Concerns\HasSpoilersParameter; use App\Dto\Concerns\PreparesData; -use App\Enums\TopAnimeFilterEnum; use App\Enums\TopReviewsTypeEnum; use App\Rules\Attributes\EnumValidation; use Illuminate\Http\JsonResponse; From 0c4afddcaca0cc49d757ae0dbdc9da172e2bef52 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 26 Jul 2024 15:49:27 +0100 Subject: [PATCH 08/31] fixed tests --- tests/Integration/TopControllerTest.php | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/Integration/TopControllerTest.php b/tests/Integration/TopControllerTest.php index f445066d..4906dd13 100644 --- a/tests/Integration/TopControllerTest.php +++ b/tests/Integration/TopControllerTest.php @@ -7,6 +7,21 @@ use App\Testing\ScoutFlush; use App\Testing\SyntheticMongoDbTransaction; use Illuminate\Database\Eloquent\Factories\Sequence; +use Illuminate\Support\Carbon; +use Jikan\Exception\BadResponseException; +use Jikan\Exception\ParserException; +use Jikan\Model\Anime\AnimeReview; +use Jikan\Model\Anime\AnimeReviewScores; +use Jikan\Model\Manga\MangaReview; +use Jikan\Model\Manga\MangaReviewScores; +use Jikan\Model\Resource\UserImageResource\UserImageResource; +use Jikan\Model\Reviews\Reactions; +use Jikan\Model\Reviews\Reviewer; +use Jikan\Model\Reviews\Reviews; +use Jikan\MyAnimeList\MalClient; +use Jikan\Parser\Reviews\AnimeReviewParser; +use Jikan\Parser\Reviews\MangaReviewParser; +use Jikan\Parser\Reviews\ReviewsParser; use Tests\TestCase; class TopControllerTest extends TestCase @@ -304,9 +319,21 @@ public function test404() * @dataProvider topReviewTypeParametersProvider * @param $params * @return void + * @throws BadResponseException + * @throws ParserException */ public function testTopReviews($params) { + $jikanParser = \Mockery::mock(MalClient::class)->makePartial(); + + $reviewsParser = \Mockery::mock(ReviewsParser::class)->makePartial(); + $reviewsParser->allows()->getReviews()->andReturn([]); + $reviewsParser->allows()->hasNextPage()->andReturn(false); + $reviewsFacade = Reviews::fromParser($reviewsParser); + + /** @noinspection PhpParamsInspection */ + $jikanParser->allows()->getReviews(\Mockery::any())->andReturn($reviewsFacade); + $this->app->instance('JikanParser', $jikanParser); $this->getJsonResponse($params,"/v4/top/reviews"); $this->seeStatusCode(200); } From 08af55545bdd5ec0018b151c1715761985022194 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 26 Jul 2024 15:52:44 +0100 Subject: [PATCH 09/31] cleanup --- tests/Integration/TopControllerTest.php | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/Integration/TopControllerTest.php b/tests/Integration/TopControllerTest.php index 4906dd13..22f0e462 100644 --- a/tests/Integration/TopControllerTest.php +++ b/tests/Integration/TopControllerTest.php @@ -7,20 +7,10 @@ use App\Testing\ScoutFlush; use App\Testing\SyntheticMongoDbTransaction; use Illuminate\Database\Eloquent\Factories\Sequence; -use Illuminate\Support\Carbon; use Jikan\Exception\BadResponseException; use Jikan\Exception\ParserException; -use Jikan\Model\Anime\AnimeReview; -use Jikan\Model\Anime\AnimeReviewScores; -use Jikan\Model\Manga\MangaReview; -use Jikan\Model\Manga\MangaReviewScores; -use Jikan\Model\Resource\UserImageResource\UserImageResource; -use Jikan\Model\Reviews\Reactions; -use Jikan\Model\Reviews\Reviewer; use Jikan\Model\Reviews\Reviews; use Jikan\MyAnimeList\MalClient; -use Jikan\Parser\Reviews\AnimeReviewParser; -use Jikan\Parser\Reviews\MangaReviewParser; use Jikan\Parser\Reviews\ReviewsParser; use Tests\TestCase; From 017bea748a6932b41c2dcb72c94f414cf27ce20c Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 26 Jul 2024 17:47:37 +0100 Subject: [PATCH 10/31] Update docker-compose.yml --- docker-compose.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e1c0843d..edd17cdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ secrets: services: jikan_rest: - image: "jikanme/jikan-rest:${_JIKAN_API_VERSION:-latest}" + image: "docker.io/jikanme/jikan-rest:${_JIKAN_API_VERSION:-latest}" user: "${APP_UID:-10001}:${APP_GID:-10001}" networks: - jikan_network @@ -116,7 +116,3 @@ services: - typesense-data:/data ports: - "8108/tcp" - healthcheck: - test: [ 'CMD-SHELL', '{ ! [ -f "curl_created" ] && apt -qq update -y && apt -qq install -y curl && touch curl_created && curl -s -f http://localhost:8108/health; } || { curl -s -f http://localhost:8108/health; }' ] - interval: 5s - timeout: 2s From bef1992c2daefe0ef7e16710bf16d429da23cac0 Mon Sep 17 00:00:00 2001 From: Irfan Date: Sun, 22 Sep 2024 13:29:38 +0500 Subject: [PATCH 11/31] fix: object being passed to the parser instead of value --- app/Features/QueryTopReviewsHandler.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Features/QueryTopReviewsHandler.php b/app/Features/QueryTopReviewsHandler.php index 6eaae8b4..932cc0da 100644 --- a/app/Features/QueryTopReviewsHandler.php +++ b/app/Features/QueryTopReviewsHandler.php @@ -31,7 +31,7 @@ protected function getScraperData(string $requestFingerPrint, Collection $reques $preliminary = $requestParams->get("preliminary", true); return $this->scraperService->findList( $requestFingerPrint, - fn (MalClient $jikan, ?int $page = null) => $jikan->getReviews(new ReviewsRequest($type, $page, $spoilers, $preliminary)), + fn (MalClient $jikan, ?int $page = null) => $jikan->getReviews(new ReviewsRequest($type->value, $page, $spoilers, $preliminary)), $requestParams->get("page")); } } From 6f1e3715b0647655bb38fcad65589c7fcba4781f Mon Sep 17 00:00:00 2001 From: Irfan Date: Sun, 22 Sep 2024 13:29:47 +0500 Subject: [PATCH 12/31] update deps --- composer.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/composer.lock b/composer.lock index b7d9f1aa..572b6df8 100644 --- a/composer.lock +++ b/composer.lock @@ -4554,16 +4554,16 @@ }, { "name": "jikan-me/jikan", - "version": "v4.0.11", + "version": "v4.0.12", "source": { "type": "git", "url": "https://github.com/jikan-me/jikan.git", - "reference": "fcc8d20817ce29332b496a21652b9965c6c8196c" + "reference": "dcb47237a9407473f484bd28a5e44479cdd916fc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/jikan-me/jikan/zipball/fcc8d20817ce29332b496a21652b9965c6c8196c", - "reference": "fcc8d20817ce29332b496a21652b9965c6c8196c", + "url": "https://api.github.com/repos/jikan-me/jikan/zipball/dcb47237a9407473f484bd28a5e44479cdd916fc", + "reference": "dcb47237a9407473f484bd28a5e44479cdd916fc", "shasum": "" }, "require": { @@ -4602,7 +4602,7 @@ "description": "Jikan is an unofficial MyAnimeList API", "support": { "issues": "https://github.com/jikan-me/jikan/issues", - "source": "https://github.com/jikan-me/jikan/tree/v4.0.11" + "source": "https://github.com/jikan-me/jikan/tree/v4.0.12" }, "funding": [ { @@ -4610,7 +4610,7 @@ "type": "patreon" } ], - "time": "2024-05-30T08:15:50+00:00" + "time": "2024-09-20T22:15:42+00:00" }, { "name": "jms/metadata", @@ -13591,5 +13591,5 @@ "ext-mongodb": "*" }, "platform-dev": [], - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.3.0" } From e15ba7b5423c4ea76387476c7e66f818a602e5e0 Mon Sep 17 00:00:00 2001 From: Irfan Date: Sun, 22 Sep 2024 19:37:35 +0500 Subject: [PATCH 13/31] docs: getAnimeEpisodes: add score and remove duration #551 --- app/Http/Controllers/V4DB/AnimeController.php | 14 ++++++++------ storage/api-docs/api-docs.json | 12 +++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/app/Http/Controllers/V4DB/AnimeController.php b/app/Http/Controllers/V4DB/AnimeController.php index 161b2b66..00b901be 100644 --- a/app/Http/Controllers/V4DB/AnimeController.php +++ b/app/Http/Controllers/V4DB/AnimeController.php @@ -225,15 +225,17 @@ public function staff(AnimeStaffLookupCommand $command) * nullable=true * ), * @OA\Property( - * property="duration", - * type="integer", - * description="Episode duration in seconds", - * nullable=true - * ), - * @OA\Property( * property="aired", * type="string", * description="Aired Date ISO8601", + * nullable=true + * ), + * @OA\Property( + * property="score", + * type="number", + * description="Aggregated episode score based on MyAnimeList user voting", +* minimum="1", +* maximum="5", * nullable=true * ), * @OA\Property( diff --git a/storage/api-docs/api-docs.json b/storage/api-docs/api-docs.json index 4779da6a..8206f3f1 100644 --- a/storage/api-docs/api-docs.json +++ b/storage/api-docs/api-docs.json @@ -4480,16 +4480,18 @@ "type": "string", "nullable": true }, - "duration": { - "description": "Episode duration in seconds", - "type": "integer", - "nullable": true - }, "aired": { "description": "Aired Date ISO8601", "type": "string", "nullable": true }, + "score": { + "description": "Aggregated episode score based on MyAnimeList user voting", + "type": "number", + "maximum": "5", + "minimum": "1", + "nullable": true + }, "filler": { "description": "Filler episode", "type": "boolean" From 02f2c7b92c726a2fac011919dafbc756889a9f2a Mon Sep 17 00:00:00 2001 From: Irfan Date: Sun, 22 Sep 2024 19:45:38 +0500 Subject: [PATCH 14/31] docs: update --- app/Http/Controllers/V4DB/AnimeController.php | 4 ++-- storage/api-docs/api-docs.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Http/Controllers/V4DB/AnimeController.php b/app/Http/Controllers/V4DB/AnimeController.php index 00b901be..4e487ed0 100644 --- a/app/Http/Controllers/V4DB/AnimeController.php +++ b/app/Http/Controllers/V4DB/AnimeController.php @@ -232,8 +232,8 @@ public function staff(AnimeStaffLookupCommand $command) * ), * @OA\Property( * property="score", - * type="number", - * description="Aggregated episode score based on MyAnimeList user voting", + * type="integer", + * description="Aggregated episode score (1.00 - 5.00) based on MyAnimeList user voting", * minimum="1", * maximum="5", * nullable=true diff --git a/storage/api-docs/api-docs.json b/storage/api-docs/api-docs.json index 8206f3f1..ae5a5b32 100644 --- a/storage/api-docs/api-docs.json +++ b/storage/api-docs/api-docs.json @@ -4486,8 +4486,8 @@ "nullable": true }, "score": { - "description": "Aggregated episode score based on MyAnimeList user voting", - "type": "number", + "description": "Aggregated episode score (1.00 - 5.00) based on MyAnimeList user voting", + "type": "integer", "maximum": "5", "minimum": "1", "nullable": true From 634528a1ad2e71c59c1a1d822e971209c04fd3ad Mon Sep 17 00:00:00 2001 From: Irfan Date: Sun, 22 Sep 2024 19:45:58 +0500 Subject: [PATCH 15/31] docs: update --- app/Http/Controllers/V4DB/AnimeController.php | 2 +- storage/api-docs/api-docs.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/V4DB/AnimeController.php b/app/Http/Controllers/V4DB/AnimeController.php index 4e487ed0..abdd98df 100644 --- a/app/Http/Controllers/V4DB/AnimeController.php +++ b/app/Http/Controllers/V4DB/AnimeController.php @@ -232,7 +232,7 @@ public function staff(AnimeStaffLookupCommand $command) * ), * @OA\Property( * property="score", - * type="integer", + * type="float", * description="Aggregated episode score (1.00 - 5.00) based on MyAnimeList user voting", * minimum="1", * maximum="5", diff --git a/storage/api-docs/api-docs.json b/storage/api-docs/api-docs.json index ae5a5b32..a8bc1b78 100644 --- a/storage/api-docs/api-docs.json +++ b/storage/api-docs/api-docs.json @@ -4487,7 +4487,7 @@ }, "score": { "description": "Aggregated episode score (1.00 - 5.00) based on MyAnimeList user voting", - "type": "integer", + "type": "float", "maximum": "5", "minimum": "1", "nullable": true From 8bdcf97d4e370c8fcd74e572b2f6f10206c64a6d Mon Sep 17 00:00:00 2001 From: pushrbx Date: Tue, 15 Oct 2024 18:58:19 +0100 Subject: [PATCH 16/31] fixed test issues --- app/Dto/LookupDataCommand.php | 7 ------- app/Features/QueryTopReviewsHandler.php | 3 +-- app/Http/Resources/V4/ReviewsResource.php | 1 + app/Support/helpers.php | 9 +++++++++ 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/app/Dto/LookupDataCommand.php b/app/Dto/LookupDataCommand.php index 5949d976..9d60f2d1 100644 --- a/app/Dto/LookupDataCommand.php +++ b/app/Dto/LookupDataCommand.php @@ -4,7 +4,6 @@ use App\Concerns\HasRequestFingerprint; use App\Contracts\DataRequest; -use App\DataPipes\MapRouteParametersDataPipe; use App\Dto\Concerns\MapsRouteParameters; use Illuminate\Http\Resources\Json\JsonResource; use Illuminate\Http\Resources\Json\ResourceCollection; @@ -13,12 +12,6 @@ use Spatie\LaravelData\Attributes\Validation\Numeric; use Spatie\LaravelData\Attributes\Validation\Required; use Spatie\LaravelData\Data; -use Spatie\LaravelData\DataPipeline; -use Spatie\LaravelData\DataPipes\AuthorizedDataPipe; -use Spatie\LaravelData\DataPipes\CastPropertiesDataPipe; -use Spatie\LaravelData\DataPipes\DefaultValuesDataPipe; -use Spatie\LaravelData\DataPipes\MapPropertiesDataPipe; -use Spatie\LaravelData\DataPipes\ValidatePropertiesDataPipe; /** * Base class for all requests/commands which are for looking up things by id. diff --git a/app/Features/QueryTopReviewsHandler.php b/app/Features/QueryTopReviewsHandler.php index 932cc0da..a967d992 100644 --- a/app/Features/QueryTopReviewsHandler.php +++ b/app/Features/QueryTopReviewsHandler.php @@ -7,7 +7,6 @@ use App\Support\CachedData; use Illuminate\Http\JsonResponse; use Illuminate\Support\Collection; -use Jikan\Helper\Constants; use Jikan\MyAnimeList\MalClient; use Jikan\Request\Reviews\ReviewsRequest; @@ -31,7 +30,7 @@ protected function getScraperData(string $requestFingerPrint, Collection $reques $preliminary = $requestParams->get("preliminary", true); return $this->scraperService->findList( $requestFingerPrint, - fn (MalClient $jikan, ?int $page = null) => $jikan->getReviews(new ReviewsRequest($type->value, $page, $spoilers, $preliminary)), + fn (MalClient $jikan, ?int $page = null) => $jikan->getReviews(new ReviewsRequest(ensureEnumPrimitiveValue($type), $page, $spoilers, $preliminary)), $requestParams->get("page")); } } diff --git a/app/Http/Resources/V4/ReviewsResource.php b/app/Http/Resources/V4/ReviewsResource.php index 256beb20..0bca9e0f 100644 --- a/app/Http/Resources/V4/ReviewsResource.php +++ b/app/Http/Resources/V4/ReviewsResource.php @@ -3,6 +3,7 @@ namespace App\Http\Resources\V4; use Illuminate\Http\Resources\Json\JsonResource; +use OpenApi\Annotations as OA; class ReviewsResource extends JsonResource { diff --git a/app/Support/helpers.php b/app/Support/helpers.php index dc9094df..df69542d 100644 --- a/app/Support/helpers.php +++ b/app/Support/helpers.php @@ -114,3 +114,12 @@ function cache(...$arguments) return app('cache')->put(key($arguments[0]), reset($arguments[0]), $arguments[1] ?? null); } } + +if (!function_exists("ensureEnumPrimitiveValue")) { + function ensureEnumPrimitiveValue(int|string|bool|float|null|\Spatie\Enum\Laravel\Enum $value): mixed { + if ($value instanceof \Spatie\Enum\Laravel\Enum) { + return $value->value; + } + return $value; + } +} From 775e2eb5aafa2cd64fe856e8f85fe2bfa1dee881 Mon Sep 17 00:00:00 2001 From: purarue <7804791+purarue@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:14:21 -0700 Subject: [PATCH 17/31] bug: fix index URL to new username --- COMMANDS.MD | 4 ++-- README.MD | 1 - app/Console/Commands/Indexer/AnimeIndexer.php | 8 ++++---- app/Console/Commands/Indexer/AnimeSweepIndexer.php | 6 +++--- app/Console/Commands/Indexer/MangaIndexer.php | 8 ++++---- app/Console/Commands/Indexer/MangaSweepIndexer.php | 6 +++--- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/COMMANDS.MD b/COMMANDS.MD index 63879598..a0ebdbe1 100644 --- a/COMMANDS.MD +++ b/COMMANDS.MD @@ -66,7 +66,7 @@ Example: `cache:method queue` Since v4 uses MongoDB as a means to index cache on some endpoints, having a built cache is important since it works best for endpoints like search or top. -`Indexer:Anime` uses [https://github.com/seanbreckenridge/mal-id-cache](https://github.com/seanbreckenridge/mal-id-cache) to fetch available MAL IDs and indexes them. +`Indexer:Anime` uses [https://github.com/purarue/mal-id-cache](https://github.com/purarue/mal-id-cache) to fetch available MAL IDs and indexes them. This function only needs to be run once. Any entry's cache updating will automatically be taken care of if it's expired, and a client makes a request for that entry. @@ -90,7 +90,7 @@ This translates to running entries that previously failed to index or update, in Since v4 uses MongoDB as a means to index cache on some endpoints, having a built cache is important since it works best for endpoints like search or top. -`Indexer:Manga` uses [https://github.com/seanbreckenridge/mal-id-cache](https://github.com/seanbreckenridge/mal-id-cache) to fetch available MAL IDs and indexes them. +`Indexer:Manga` uses [https://github.com/purarue/mal-id-cache](https://github.com/purarue/mal-id-cache) to fetch available MAL IDs and indexes them. This function only needs to be run once. Any entry's cache updating will automatically be taken care of if it's expired, and a client makes a request for that entry. diff --git a/README.MD b/README.MD index 5c60c226..28e2fbe8 100644 --- a/README.MD +++ b/README.MD @@ -52,7 +52,6 @@ For any additional help, join our [Discord server](http://discord.jikan.moe/). | TypeScript | [jikants](https://github.com/Julien-Broyard/jikants) by Julien Broyard
[jikan-client](https://github.com/javi11/jikan-client) by Javier Blanco
šŸ†• **(v4)** [jikan-ts](https://github.com/tutkli/jikan-ts) by Clara Castillo | | PHP | [jikan-php](https://github.com/janvernieuwe/jikan-jikanPHP) by Jan Vernieuwe | | .NET | šŸ†• **(v4)** [Jikan.net](https://github.com/Ervie/jikan.net) by Ervie | -| Elixir | [JikanEx](https://github.com/seanbreckenridge/jikan_ex) by Sean Breckenridge | | Go | šŸ†• **(v4)** [jikan-go](https://github.com/darenliang/jikan-go) by Daren Liang
[jikan2go](https://github.com/nokusukun/jikan2go) by nokusukun | | Ruby | [Jikan.rb](https://github.com/Zerocchi/jikan.rb) by Zerocchi | | Dart | [jikan-dart](https://github.com/charafau/jikan-dart) by Rafal Wachol | diff --git a/app/Console/Commands/Indexer/AnimeIndexer.php b/app/Console/Commands/Indexer/AnimeIndexer.php index b3bbe4d0..77515daa 100644 --- a/app/Console/Commands/Indexer/AnimeIndexer.php +++ b/app/Console/Commands/Indexer/AnimeIndexer.php @@ -67,7 +67,7 @@ public function handle() $index = (int)$index; $delay = (int)$delay; - $this->info("Info: AnimeIndexer uses seanbreckenridge/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n"); + $this->info("Info: AnimeIndexer uses purarue/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n"); if ($failed && Storage::exists('indexer/indexer_anime.save')) { $this->ids = $this->loadFailedMalIds(); @@ -140,14 +140,14 @@ public function handle() /** * @return array - * @url https://github.com/seanbreckenridge/mal-id-cache + * @url https://github.com/purarue/mal-id-cache */ private function fetchMalIds() : array { - $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/anime_cache.json...\n"); + $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/anime_cache.json...\n"); $ids = json_decode( - file_get_contents('https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/anime_cache.json'), + file_get_contents('https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/anime_cache.json'), true ); diff --git a/app/Console/Commands/Indexer/AnimeSweepIndexer.php b/app/Console/Commands/Indexer/AnimeSweepIndexer.php index ebfdffe9..f29954ea 100644 --- a/app/Console/Commands/Indexer/AnimeSweepIndexer.php +++ b/app/Console/Commands/Indexer/AnimeSweepIndexer.php @@ -74,14 +74,14 @@ public function handle() /** * @return array - * @url https://github.com/seanbreckenridge/mal-id-cache + * @url https://github.com/purarue/mal-id-cache */ private function fetchMalIds(): array { - $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/anime_cache.json...\n"); + $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/anime_cache.json...\n"); $ids = json_decode( - file_get_contents('https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/anime_cache.json'), + file_get_contents('https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/anime_cache.json'), true ); diff --git a/app/Console/Commands/Indexer/MangaIndexer.php b/app/Console/Commands/Indexer/MangaIndexer.php index ee07c1fa..e1821e84 100644 --- a/app/Console/Commands/Indexer/MangaIndexer.php +++ b/app/Console/Commands/Indexer/MangaIndexer.php @@ -67,7 +67,7 @@ public function handle() $index = (int)$index; $delay = (int)$delay; - $this->info("Info: MangaIndexer uses seanbreckenridge/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n"); + $this->info("Info: MangaIndexer uses purarue/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n"); if ($failed && Storage::exists('indexer/indexer_manga.save')) { $this->ids = $this->loadFailedMalIds(); @@ -140,14 +140,14 @@ public function handle() /** * @return array - * @url https://github.com/seanbreckenridge/mal-id-cache + * @url https://github.com/purarue/mal-id-cache */ private function fetchMalIds() : array { - $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json...\n"); + $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/manga_cache.json...\n"); $ids = json_decode( - file_get_contents('https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json'), + file_get_contents('https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/manga_cache.json'), true ); diff --git a/app/Console/Commands/Indexer/MangaSweepIndexer.php b/app/Console/Commands/Indexer/MangaSweepIndexer.php index 09dfdb39..1f45e3e9 100644 --- a/app/Console/Commands/Indexer/MangaSweepIndexer.php +++ b/app/Console/Commands/Indexer/MangaSweepIndexer.php @@ -71,14 +71,14 @@ public function handle() /** * @return array - * @url https://github.com/seanbreckenridge/mal-id-cache + * @url https://github.com/purarue/mal-id-cache */ private function fetchMalIds(): array { - $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json...\n"); + $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/manga_cache.json...\n"); $ids = json_decode( - file_get_contents('https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json'), + file_get_contents('https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/manga_cache.json'), true ); From 705e0fbf36ce099b0b6551f52289d9076807a27f Mon Sep 17 00:00:00 2001 From: pushrbx Date: Mon, 28 Oct 2024 16:22:04 +0100 Subject: [PATCH 18/31] fixed season endpoint's continuing flag #555 --- app/Repositories/DefaultAnimeRepository.php | 38 ++++++++++++++++----- tests/Integration/SeasonControllerTest.php | 35 +++++++++++++++++++ 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/app/Repositories/DefaultAnimeRepository.php b/app/Repositories/DefaultAnimeRepository.php index ffbf103d..98436297 100644 --- a/app/Repositories/DefaultAnimeRepository.php +++ b/app/Repositories/DefaultAnimeRepository.php @@ -167,19 +167,39 @@ public function getItemsBySeason( $finalFilter['$or'][] = [ // note: this expression only works with mongodb version 5.0.0 or higher '$expr' => [ - '$lte' => [ + '$and' => [ [ - '$dateDiff' => [ - 'startDate' => [ - '$dateFromString' => [ - 'dateString' => '$aired.from' + '$lte' => [ + [ + '$dateDiff' => [ + 'startDate' => [ + '$dateFromString' => [ + 'dateString' => '$aired.from' + ] + ], + 'endDate' => new UTCDateTime($from), + 'unit' => 'month' ] ], - 'endDate' => new UTCDateTime($from), - 'unit' => 'month' - ] + 3 // there are 3 months in a season, so anything that started in 3 months or less will be included + ], ], - 3 // there are 3 months in a season, so anything that started in 3 months or less will be included + [ + '$gt' => [ + [ + '$dateDiff' => [ + 'startDate' => [ + '$dateFromString' => [ + 'dateString' => '$aired.from' + ] + ], + 'endDate' => new UTCDateTime($from), + 'unit' => 'month' + ] + ], + 0 + ] + ] ] ], 'aired.to' => null, diff --git a/tests/Integration/SeasonControllerTest.php b/tests/Integration/SeasonControllerTest.php index 6d6a752e..ffcde268 100644 --- a/tests/Integration/SeasonControllerTest.php +++ b/tests/Integration/SeasonControllerTest.php @@ -190,4 +190,39 @@ public function testShouldNotIncludeContinuingItemsByDefault() $this->assertIsArray($content["data"]); $this->assertCount(2, $content["data"]); } + + public function testShouldNotIncludeNewlyStartedSeasonOfAnimeInPreviousSeasons() + { + Carbon::setTestNow(Carbon::parse("2024-10-26")); + $f = Anime::factory(1); + $startDate = "2024-10-02"; + $carbonStartDate = Carbon::parse($startDate); + $state = $f->serializeStateDefinition([ + "aired" => new CarbonDateRange($carbonStartDate, null) + ]); + $state["aired"]["string"] = "Oct 2, 2024 to ?"; + $state["airing"] = true; + $state["status"] = "Currently Airing"; + $state["premiered"] = "Fall 2024"; + $state["mal_id"] = 54857; + $state["title"] = "Re:Zero kara Hajimeru Isekai Seikatsu 3rd Season"; + $state["episodes"] = 16; + $state["type"] = "TV"; + $state["duration"] = "23 min per ep"; + $state["score"] = 8.9; + $f->create($state); + + $content = $this->getJsonResponse([], "/v4/seasons/now?filter=tv&continuing&page=1"); + $this->seeStatusCode(200); + $this->assertCount(1, $content["data"]); + + $content = $this->getJsonResponse([], "/v4/seasons/2024/summer?filter=tv&continuing&page=1"); + $this->seeStatusCode(200); + $this->assertCount(0, $content["data"]); + + $content = $this->getJsonResponse([], "/v4/seasons/2024/spring?filter=tv&continuing&page=1"); + $this->seeStatusCode(200); + $this->assertCount(0, $content["data"]); + Carbon::setTestNow(); + } } From 9553c2e7f91d9a6e5095f438d41e51723147b2de Mon Sep 17 00:00:00 2001 From: pushrbx Date: Mon, 28 Oct 2024 16:22:48 +0100 Subject: [PATCH 19/31] fixed search index collection schema for producers --- app/Producers.php | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app/Producers.php b/app/Producers.php index f46216d8..dd5441c0 100644 --- a/app/Producers.php +++ b/app/Producers.php @@ -72,6 +72,33 @@ public function toSearchableArray(): array ]; } + public function getCollectionSchema(): array + { + return [ + 'name' => $this->searchableAs(), + 'fields' => [ + [ + 'name' => '.*', + 'type' => 'auto', + ], + [ + 'name' => 'titles', + 'type' => 'string', + 'optional' => false, + 'infix' => true, + 'sort' => true + ], + [ + 'name' => 'url', + 'type' => 'string', + 'optional' => false, + 'infix' => true, + 'sort' => true + ], + ] + ]; + } + public function typesenseQueryBy(): array { return [ From 970ff8aad4ad46040a04c9d4fb4ed74278a81354 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Mon, 4 Nov 2024 21:09:02 +0000 Subject: [PATCH 20/31] removed redundant imports --- app/Console/Commands/Indexer/AnimeIndexer.php | 1 - app/Features/QuerySpecificAnimeSeasonHandler.php | 2 -- 2 files changed, 3 deletions(-) diff --git a/app/Console/Commands/Indexer/AnimeIndexer.php b/app/Console/Commands/Indexer/AnimeIndexer.php index 3e03f302..b70ae429 100644 --- a/app/Console/Commands/Indexer/AnimeIndexer.php +++ b/app/Console/Commands/Indexer/AnimeIndexer.php @@ -2,7 +2,6 @@ namespace App\Console\Commands\Indexer; -use App\Exceptions\Console\CommandAlreadyRunningException; use App\Exceptions\Console\FileNotFoundException; use Illuminate\Console\Command; use Illuminate\Support\Facades\Storage; diff --git a/app/Features/QuerySpecificAnimeSeasonHandler.php b/app/Features/QuerySpecificAnimeSeasonHandler.php index 1ec68210..e073fa0f 100644 --- a/app/Features/QuerySpecificAnimeSeasonHandler.php +++ b/app/Features/QuerySpecificAnimeSeasonHandler.php @@ -3,8 +3,6 @@ namespace App\Features; use App\Dto\QuerySpecificAnimeSeasonCommand; -use App\Enums\AnimeSeasonEnum; -use App\Enums\AnimeStatusEnum; use App\Enums\AnimeTypeEnum; use Illuminate\Contracts\Database\Query\Builder; use Illuminate\Support\Carbon; From e0cc44495bb6a32c7e7658b49f35d43b5775bb89 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Mon, 4 Nov 2024 21:09:21 +0000 Subject: [PATCH 21/31] added incremental indexer --- .../Commands/Indexer/IncrementalIndexer.php | 183 ++++++++++++++++++ app/Console/Kernel.php | 3 +- composer.json | 1 + container-setup.sh | 5 + 4 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 app/Console/Commands/Indexer/IncrementalIndexer.php diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php new file mode 100644 index 00000000..bc618e0d --- /dev/null +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -0,0 +1,183 @@ + ['The media type to index.', 'Valid values: anime, manga, character, people'] + ]; + } + + public function handle(): int + { + $validator = Validator::make( + [ + 'mediaType' => $this->argument('mediaType'), + 'delay' => $this->option('delay'), + 'resume' => $this->option('resume') ?? false, + 'failed' => $this->option('failed') ?? false + ], + [ + 'mediaType' => 'required|in:anime,manga,character,people', + 'delay' => 'integer|min:1', + 'resume' => 'bool|prohibited_with:failed', + 'failed' => 'bool|prohibited_with:resume' + ] + ); + + if ($validator->fails()) { + $this->error($validator->errors()->toJson()); + return 1; + } + + $this->trap(SIGTERM, fn () => $this->cancelled = true); + + $resume = $this->option('resume') ?? false; + $onlyFailed = $this->option('failed') ?? false; + $existingIdsHash = ""; + $existingIdsRaw = ""; + /** + * @var $mediaTypes array + */ + $mediaTypes = $this->argument("mediaType"); + + foreach ($mediaTypes as $mediaType) + { + $idsToFetch = []; + $failedIds = []; + $success = []; + + if ($onlyFailed && Storage::exists("indexer/incremental/{$mediaType}_failed.json")) + { + $idsToFetch["sfw"] = json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json")); + } + else + { + if (Storage::exists("indexer/incremental/$mediaType.json")) + { + $existingIdsRaw = Storage::get("indexer/incremental/$mediaType.json"); + $existingIdsHash = sha1($existingIdsRaw); + } + + if ($this->cancelled) + { + return 127; + } + + $newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json"); + $newIdsHash = sha1($newIdsRaw); + + /** @noinspection PhpConditionAlreadyCheckedInspection */ + if ($this->cancelled) + { + return 127; + } + + if ($newIdsHash !== $existingIdsHash) + { + $newIds = json_decode($newIdsRaw, true); + $existingIds = json_decode($existingIdsRaw, true); + + if (is_null($existingIds) || count($existingIds) === 0) + { + $idsToFetch = $newIds; + } + else + { + foreach (["sfw", "nsfw"] as $t) + { + $idsToFetch[$t] = array_diff($existingIds[$t], $newIds[$t]); + } + } + + Storage::put("indexer/incremental/$mediaType.json.tmp", $newIdsRaw); + } + } + + $idCount = count($idsToFetch); + if ($idCount > 0) + { + $index = 0; + if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save")) + { + $index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save"); + $this->info("Resuming from index: $index"); + } + + if ($index > 0 && !isset($this->ids[$index])) { + $index = 0; + $this->warn('Invalid index; set back to 0'); + } + + Storage::put("indexer/incremental/{$mediaType}_resume.save", 0); + + $this->info("$idCount $mediaType entries available"); + $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); + for ($i = $index; $i <= ($idCount - 1); $i++) + { + if ($this->cancelled) + { + return 127; + } + + $id = $ids[$index]; + + $url = env('APP_URL') . "/v4/anime/$id"; + $this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]"); + + try + { + $response = json_decode(file_get_contents($url), true); + if (isset($response['error']) && $response['status'] != 404) + { + $this->error("[SKIPPED] Failed to fetch $url - {$response['error']}"); + } + } + catch (\Exception) + { + $this->warn("[SKIPPED] Failed to fetch $url"); + $failedIds[] = $id; + Storage::put("indexer/incremental/$mediaType.failed", json_encode($failedIds)); + } + + $success[] = $id; + Storage::put("indexer/incremental/{$mediaType}_resume.save", $index); + } + + Storage::delete("indexer/incremental/{$mediaType}_resume.save"); + $this->info("--- Indexing of $mediaType is complete."); + $this->info(count($success) . ' entries indexed or updated.'); + if (count($failedIds) > 0) + { + $this->info(count($failedIds) . ' entries failed to index or update. Re-run with --failed to requeue failed entries only.'); + } + // finalize the latest state + Storage::move("indexer/incremental/$mediaType.json.tmp", "indexer/incremental/$mediaType.json"); + } + } + + return 0; + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 22c9b19c..eafe5553 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -24,7 +24,8 @@ class Kernel extends ConsoleKernel Indexer\GenreIndexer::class, Indexer\ProducersIndexer::class, Indexer\AnimeSweepIndexer::class, - Indexer\MangaSweepIndexer::class + Indexer\MangaSweepIndexer::class, + Indexer\IncrementalIndexer::class ]; /** diff --git a/composer.json b/composer.json index 8234697c..55e6a704 100644 --- a/composer.json +++ b/composer.json @@ -14,6 +14,7 @@ "php": "^8.1", "ext-json": "*", "ext-mongodb": "*", + "ext-pcntl": "*", "amphp/http-client": "^4.6", "danielmewes/php-rql": "dev-master", "darkaonline/swagger-lume": "^9.0", diff --git a/container-setup.sh b/container-setup.sh index 89c39c61..dd56f6e9 100755 --- a/container-setup.sh +++ b/container-setup.sh @@ -34,6 +34,7 @@ display_help() { echo "stop Stop Jikan API" echo "validate-prereqs Validate pre-reqs installed (docker, docker-compose)" echo "execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)" + echo "index-incrementally Executes the incremental indexers for each media type. (anime, manga, character, people)" echo "" } @@ -168,6 +169,10 @@ case "$1" in $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:producers echo "Indexing done!" ;; + "index-incrementally") + echo "Indexing..." + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:incremental anime manga character people + echo "Indexing done!" *) echo "No command specified, displaying help" display_help From 6568ff264e1b14cb43b88f7712b260b86893fa50 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Mon, 4 Nov 2024 21:22:07 +0000 Subject: [PATCH 22/31] added docs for the new command --- COMMANDS.MD | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/COMMANDS.MD b/COMMANDS.MD index b0656eb4..e07b6fe8 100644 --- a/COMMANDS.MD +++ b/COMMANDS.MD @@ -14,14 +14,14 @@ For an entire list of commands, you can run `php artisan list` - [Indexer](#indexer) - [Anime](#indexer-anime) - [Manga](#indexer-manga) - + ## Commands ### Serve Command: `serve` Example: `php artisan serve` -Serve the application on the PHP development server +Serve the application on the PHP development server ### Queue @@ -98,7 +98,7 @@ This function only needs to be run once. Any entry's cache updating will automat Command: ``` -indexer:anime +indexer:manga {--failed : Run only entries that failed to index last time} {--resume : Resume from the last position} {--reverse : Start from the end of the array} @@ -109,3 +109,16 @@ indexer:anime Example: `indexer:manga` This simply translates to running the indexer without any additional configuration. + +#### Indexer: Incremental +Incrementally indexes media entries from MAL. +This command will compare the latest version of MAL ids from the [mal_id_cache](https://github.com/purarue/mal-id-cache) +github repository and compares them with the downloaded ids from the previous run. If no ids found from the previous run, a full indexing session is started. + +Command: +``` +indexer:incremental {mediaType*} + {--failed : Run only entries that failed to index last time} + {--resume : Resume from the last position} + {--delay=3 : Set a delay between requests} +``` From 51489501258ac750623f89f56860eedce67484d9 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Mon, 4 Nov 2024 23:11:49 +0000 Subject: [PATCH 23/31] added docs for the new command --- container_usage.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/container_usage.md b/container_usage.md index bce6bd74..a9cbb5bd 100644 --- a/container_usage.md +++ b/container_usage.md @@ -16,6 +16,9 @@ This will: > **Note**: The script supports both `docker` and `podman`. In case of `podman` please bare in mind that sometimes the container name resolution doesn't work on the container network. > In those cases you might have to install `aardvark-dns` package. On `Arch Linux` podman uses `netavark` network by default (in 2023) so you will need to install the before mentioned package. +> **Note 2**: The script will start the jikan API, but if you start it for the first time, it won't have any data in it! +> You will have to run the indexers through artisan to have data. See ["Running the indexer with the script"](#running-the-indexer-with-the-script) section. + The script has the following prerequisites and will notify you if these are not present: - git @@ -36,6 +39,7 @@ start Start Jikan API (mongodb, typesense, redis, jikan-api wor stop Stop Jikan API validate-prereqs Validate pre-reqs installed (docker, docker-compose) execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days) +index-incrementally Executes the incremental indexers for each media type. (anime, manga, character, people) ``` ### Running the indexer with the script From 6c2d2d3e9da31f4521e037805a4b687b94bb50af Mon Sep 17 00:00:00 2001 From: pushrbx Date: Tue, 5 Nov 2024 09:25:52 +0000 Subject: [PATCH 24/31] refactored the code for better readability --- .../Commands/Indexer/IncrementalIndexer.php | 241 +++++++++++------- 1 file changed, 142 insertions(+), 99 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index bc618e0d..8e09ecf6 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -30,8 +30,139 @@ protected function promptForMissingArgumentsUsing(): array ]; } + private function getExistingIds(string $mediaType): array + { + $existingIdsHash = ""; + $existingIdsRaw = ""; + + if (Storage::exists("indexer/incremental/$mediaType.json")) + { + $existingIdsRaw = Storage::get("indexer/incremental/$mediaType.json"); + $existingIdsHash = sha1($existingIdsRaw); + } + + return [$existingIdsHash, $existingIdsRaw]; + } + + private function getIdsToFetch(string $mediaType): array + { + $idsToFetch = []; + [$existingIdsHash, $existingIdsRaw] = $this->getExistingIds($mediaType); + + if ($this->cancelled) + { + return []; + } + + $newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json"); + $newIdsHash = sha1($newIdsRaw); + + /** @noinspection PhpConditionAlreadyCheckedInspection */ + if ($this->cancelled) + { + return []; + } + + if ($newIdsHash !== $existingIdsHash) + { + $newIds = json_decode($newIdsRaw, true); + $existingIds = json_decode($existingIdsRaw, true); + + if (is_null($existingIds) || count($existingIds) === 0) + { + $idsToFetch = $newIds; + } + else + { + foreach (["sfw", "nsfw"] as $t) + { + $idsToFetch[$t] = array_diff($existingIds[$t], $newIds[$t]); + } + } + + Storage::put("indexer/incremental/$mediaType.json.tmp", $newIdsRaw); + } + + return $idsToFetch; + } + + private function getFailedIdsToFetch(string $mediaType): array + { + return json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json")); + } + + private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): void + { + $index = 0; + $success = []; + $failedIds = []; + $idCount = count($idsToFetch); + if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save")) + { + $index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save"); + $this->info("Resuming from index: $index"); + } + + if ($index > 0 && !isset($this->ids[$index])) + { + $index = 0; + $this->warn('Invalid index; set back to 0'); + } + + Storage::put("indexer/incremental/{$mediaType}_resume.save", 0); + + $this->info("$idCount $mediaType entries available"); + $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); + + for ($i = $index; $i <= ($idCount - 1); $i++) + { + if ($this->cancelled) + { + return; + } + + $id = $ids[$index]; + + $url = env('APP_URL') . "/v4/anime/$id"; + $this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]"); + + try + { + $response = json_decode(file_get_contents($url), true); + if (!isset($response['error']) || $response['status'] == 404) + { + continue; + } + + $this->error("[SKIPPED] Failed to fetch $url - {$response['error']}"); + } + catch (\Exception) + { + $this->warn("[SKIPPED] Failed to fetch $url"); + $failedIds[] = $id; + Storage::put("indexer/incremental/$mediaType.failed", json_encode($failedIds)); + } + + $success[] = $id; + Storage::put("indexer/incremental/{$mediaType}_resume.save", $index); + } + + Storage::delete("indexer/incremental/{$mediaType}_resume.save"); + + $this->info("--- Indexing of $mediaType is complete."); + $this->info(count($success) . ' entries indexed or updated.'); + if (count($failedIds) > 0) + { + $this->info(count($failedIds) . ' entries failed to index or update. Re-run with --failed to requeue failed entries only.'); + } + + // finalize the latest state + Storage::move("indexer/incremental/$mediaType.json.tmp", "indexer/incremental/$mediaType.json"); + } + public function handle(): int { + // validate inputs $validator = Validator::make( [ 'mediaType' => $this->argument('mediaType'), @@ -52,12 +183,12 @@ public function handle(): int return 1; } + // we want to handle signals from the OS $this->trap(SIGTERM, fn () => $this->cancelled = true); $resume = $this->option('resume') ?? false; $onlyFailed = $this->option('failed') ?? false; - $existingIdsHash = ""; - $existingIdsRaw = ""; + /** * @var $mediaTypes array */ @@ -66,116 +197,28 @@ public function handle(): int foreach ($mediaTypes as $mediaType) { $idsToFetch = []; - $failedIds = []; - $success = []; + // if "--failed" option is specified just run the failed ones if ($onlyFailed && Storage::exists("indexer/incremental/{$mediaType}_failed.json")) { - $idsToFetch["sfw"] = json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json")); + $idsToFetch["sfw"] = $this->getFailedIdsToFetch($mediaType); } else { - if (Storage::exists("indexer/incremental/$mediaType.json")) - { - $existingIdsRaw = Storage::get("indexer/incremental/$mediaType.json"); - $existingIdsHash = sha1($existingIdsRaw); - } - - if ($this->cancelled) - { - return 127; - } - - $newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json"); - $newIdsHash = sha1($newIdsRaw); - - /** @noinspection PhpConditionAlreadyCheckedInspection */ - if ($this->cancelled) - { - return 127; - } - - if ($newIdsHash !== $existingIdsHash) - { - $newIds = json_decode($newIdsRaw, true); - $existingIds = json_decode($existingIdsRaw, true); - - if (is_null($existingIds) || count($existingIds) === 0) - { - $idsToFetch = $newIds; - } - else - { - foreach (["sfw", "nsfw"] as $t) - { - $idsToFetch[$t] = array_diff($existingIds[$t], $newIds[$t]); - } - } - - Storage::put("indexer/incremental/$mediaType.json.tmp", $newIdsRaw); - } + $idsToFetch = $this->getIdsToFetch($mediaType); } $idCount = count($idsToFetch); - if ($idCount > 0) + if ($idCount == 0) { - $index = 0; - if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save")) - { - $index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save"); - $this->info("Resuming from index: $index"); - } - - if ($index > 0 && !isset($this->ids[$index])) { - $index = 0; - $this->warn('Invalid index; set back to 0'); - } - - Storage::put("indexer/incremental/{$mediaType}_resume.save", 0); - - $this->info("$idCount $mediaType entries available"); - $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); - for ($i = $index; $i <= ($idCount - 1); $i++) - { - if ($this->cancelled) - { - return 127; - } - - $id = $ids[$index]; - - $url = env('APP_URL') . "/v4/anime/$id"; - $this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]"); - - try - { - $response = json_decode(file_get_contents($url), true); - if (isset($response['error']) && $response['status'] != 404) - { - $this->error("[SKIPPED] Failed to fetch $url - {$response['error']}"); - } - } - catch (\Exception) - { - $this->warn("[SKIPPED] Failed to fetch $url"); - $failedIds[] = $id; - Storage::put("indexer/incremental/$mediaType.failed", json_encode($failedIds)); - } - - $success[] = $id; - Storage::put("indexer/incremental/{$mediaType}_resume.save", $index); - } - - Storage::delete("indexer/incremental/{$mediaType}_resume.save"); - $this->info("--- Indexing of $mediaType is complete."); - $this->info(count($success) . ' entries indexed or updated.'); - if (count($failedIds) > 0) + if ($this->cancelled) { - $this->info(count($failedIds) . ' entries failed to index or update. Re-run with --failed to requeue failed entries only.'); + return 127; } - // finalize the latest state - Storage::move("indexer/incremental/$mediaType.json.tmp", "indexer/incremental/$mediaType.json"); + continue; } + + $this->fetchIds($mediaType, $idsToFetch, $resume); } return 0; From 15bd63fdc7eae10845add3a552b6d8bf8c09e78a Mon Sep 17 00:00:00 2001 From: pushrbx Date: Tue, 5 Nov 2024 09:30:13 +0000 Subject: [PATCH 25/31] fixed media type choices and url for indexing --- app/Console/Commands/Indexer/IncrementalIndexer.php | 6 +++--- container-setup.sh | 2 +- container_usage.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index 8e09ecf6..0ec05d4e 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -26,7 +26,7 @@ class IncrementalIndexer extends Command protected function promptForMissingArgumentsUsing(): array { return [ - 'mediaType' => ['The media type to index.', 'Valid values: anime, manga, character, people'] + 'mediaType' => ['The media type to index.', 'Valid values: anime, manga, characters, people'] ]; } @@ -123,7 +123,7 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v $id = $ids[$index]; - $url = env('APP_URL') . "/v4/anime/$id"; + $url = env('APP_URL') . "/v4/$mediaType/$id"; $this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]"); try @@ -171,7 +171,7 @@ public function handle(): int 'failed' => $this->option('failed') ?? false ], [ - 'mediaType' => 'required|in:anime,manga,character,people', + 'mediaType' => 'required|in:anime,manga,characters,people', 'delay' => 'integer|min:1', 'resume' => 'bool|prohibited_with:failed', 'failed' => 'bool|prohibited_with:resume' diff --git a/container-setup.sh b/container-setup.sh index dd56f6e9..f47c076f 100755 --- a/container-setup.sh +++ b/container-setup.sh @@ -34,7 +34,7 @@ display_help() { echo "stop Stop Jikan API" echo "validate-prereqs Validate pre-reqs installed (docker, docker-compose)" echo "execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)" - echo "index-incrementally Executes the incremental indexers for each media type. (anime, manga, character, people)" + echo "index-incrementally Executes the incremental indexers for each media type. (anime, manga, characters, people)" echo "" } diff --git a/container_usage.md b/container_usage.md index a9cbb5bd..8de9b8d2 100644 --- a/container_usage.md +++ b/container_usage.md @@ -39,7 +39,7 @@ start Start Jikan API (mongodb, typesense, redis, jikan-api wor stop Stop Jikan API validate-prereqs Validate pre-reqs installed (docker, docker-compose) execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days) -index-incrementally Executes the incremental indexers for each media type. (anime, manga, character, people) +index-incrementally Executes the incremental indexers for each media type. (anime, manga, characters, people) ``` ### Running the indexer with the script From cc5c12ff390ab7c16e70f7a712141331ae58283d Mon Sep 17 00:00:00 2001 From: pushrbx Date: Tue, 5 Nov 2024 16:30:41 +0000 Subject: [PATCH 26/31] removed "characters" and "people" media types from incremental indexer command --- app/Console/Commands/Indexer/IncrementalIndexer.php | 4 ++-- container-setup.sh | 4 ++-- container_usage.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index 0ec05d4e..fe532a06 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -26,7 +26,7 @@ class IncrementalIndexer extends Command protected function promptForMissingArgumentsUsing(): array { return [ - 'mediaType' => ['The media type to index.', 'Valid values: anime, manga, characters, people'] + 'mediaType' => ['The media type to index.', 'Valid values: anime, manga'] ]; } @@ -171,7 +171,7 @@ public function handle(): int 'failed' => $this->option('failed') ?? false ], [ - 'mediaType' => 'required|in:anime,manga,characters,people', + 'mediaType' => 'required|in:anime,manga', 'delay' => 'integer|min:1', 'resume' => 'bool|prohibited_with:failed', 'failed' => 'bool|prohibited_with:resume' diff --git a/container-setup.sh b/container-setup.sh index f47c076f..3cc1f5c0 100755 --- a/container-setup.sh +++ b/container-setup.sh @@ -34,7 +34,7 @@ display_help() { echo "stop Stop Jikan API" echo "validate-prereqs Validate pre-reqs installed (docker, docker-compose)" echo "execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days)" - echo "index-incrementally Executes the incremental indexers for each media type. (anime, manga, characters, people)" + echo "index-incrementally Executes the incremental indexers for each media type. (anime, manga)" echo "" } @@ -171,7 +171,7 @@ case "$1" in ;; "index-incrementally") echo "Indexing..." - $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:incremental anime manga character people + $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:incremental anime manga echo "Indexing done!" *) echo "No command specified, displaying help" diff --git a/container_usage.md b/container_usage.md index 8de9b8d2..f20e1103 100644 --- a/container_usage.md +++ b/container_usage.md @@ -39,7 +39,7 @@ start Start Jikan API (mongodb, typesense, redis, jikan-api wor stop Stop Jikan API validate-prereqs Validate pre-reqs installed (docker, docker-compose) execute-indexers Execute the indexers, which will scrape and index data from MAL. (Notice: This can take days) -index-incrementally Executes the incremental indexers for each media type. (anime, manga, characters, people) +index-incrementally Executes the incremental indexers for each media type. (anime, manga) ``` ### Running the indexer with the script From a87b4617d007e7b25fe85c327b4f9ce3f20087b4 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 8 Nov 2024 19:30:00 +0000 Subject: [PATCH 27/31] fixed some issues with IncrementalIndexer --- .../Commands/Indexer/IncrementalIndexer.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index fe532a06..eb076ac0 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -103,7 +103,9 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v $this->info("Resuming from index: $index"); } - if ($index > 0 && !isset($this->ids[$index])) + $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); + + if ($index > 0 && !isset($ids[$index])) { $index = 0; $this->warn('Invalid index; set back to 0'); @@ -112,7 +114,6 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v Storage::put("indexer/incremental/{$mediaType}_resume.save", 0); $this->info("$idCount $mediaType entries available"); - $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); for ($i = $index; $i <= ($idCount - 1); $i++) { @@ -184,7 +185,7 @@ public function handle(): int } // we want to handle signals from the OS - $this->trap(SIGTERM, fn () => $this->cancelled = true); + $this->trap([SIGTERM, SIGQUIT, SIGINT], fn () => $this->cancelled = true); $resume = $this->option('resume') ?? false; $onlyFailed = $this->option('failed') ?? false; @@ -208,13 +209,14 @@ public function handle(): int $idsToFetch = $this->getIdsToFetch($mediaType); } + if ($this->cancelled) + { + return 127; + } + $idCount = count($idsToFetch); - if ($idCount == 0) + if ($idCount === 0) { - if ($this->cancelled) - { - return 127; - } continue; } From 3ec81f2eb1d3dd0508ead78d1190ac78ba80619f Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 8 Nov 2024 19:31:51 +0000 Subject: [PATCH 28/31] updated COMMANDS.md --- COMMANDS.MD | 1 + 1 file changed, 1 insertion(+) diff --git a/COMMANDS.MD b/COMMANDS.MD index e07b6fe8..e1be3837 100644 --- a/COMMANDS.MD +++ b/COMMANDS.MD @@ -14,6 +14,7 @@ For an entire list of commands, you can run `php artisan list` - [Indexer](#indexer) - [Anime](#indexer-anime) - [Manga](#indexer-manga) + - [Incremental](#indexer-incremental) ## Commands From 2272ed17c4f6a99269b031c9a696b1ab5efe89ff Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 8 Nov 2024 19:36:45 +0000 Subject: [PATCH 29/31] added typesense collection schema for Magazine model --- app/Magazine.php | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app/Magazine.php b/app/Magazine.php index 513fdb41..8c23553b 100644 --- a/app/Magazine.php +++ b/app/Magazine.php @@ -72,6 +72,33 @@ public function toSearchableArray(): array ]; } + public function getCollectionSchema(): array + { + return [ + 'name' => $this->searchableAs(), + 'fields' => [ + [ + 'name' => '.*', + 'type' => 'auto', + ], + [ + 'name' => 'titles', + 'type' => 'string', + 'optional' => false, + 'infix' => true, + 'sort' => true + ], + [ + 'name' => 'url', + 'type' => 'string', + 'optional' => false, + 'infix' => true, + 'sort' => true + ], + ] + ]; + } + public function typesenseQueryBy(): array { return [ From 8c6398dd58c1308a4683eec55de74ecb4805c825 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Fri, 8 Nov 2024 19:46:07 +0000 Subject: [PATCH 30/31] revert changes to magazine model --- app/Magazine.php | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/app/Magazine.php b/app/Magazine.php index 8c23553b..513fdb41 100644 --- a/app/Magazine.php +++ b/app/Magazine.php @@ -72,33 +72,6 @@ public function toSearchableArray(): array ]; } - public function getCollectionSchema(): array - { - return [ - 'name' => $this->searchableAs(), - 'fields' => [ - [ - 'name' => '.*', - 'type' => 'auto', - ], - [ - 'name' => 'titles', - 'type' => 'string', - 'optional' => false, - 'infix' => true, - 'sort' => true - ], - [ - 'name' => 'url', - 'type' => 'string', - 'optional' => false, - 'infix' => true, - 'sort' => true - ], - ] - ]; - } - public function typesenseQueryBy(): array { return [ From 2fa3958ad97684f74faa61728ba62e70e2149d0a Mon Sep 17 00:00:00 2001 From: pushrbx Date: Sat, 9 Nov 2024 09:42:56 +0000 Subject: [PATCH 31/31] fixed error in container-setup.sh --- container-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/container-setup.sh b/container-setup.sh index 3cc1f5c0..3328af23 100755 --- a/container-setup.sh +++ b/container-setup.sh @@ -173,6 +173,7 @@ case "$1" in echo "Indexing..." $DOCKER_COMPOSE_CMD -p "$DOCKER_COMPOSE_PROJECT_NAME" exec jikan_rest php /app/artisan indexer:incremental anime manga echo "Indexing done!" + ;; *) echo "No command specified, displaying help" display_help