Fix: Refactor import functions and avoid async db access, avoids sqlite database lock

Author: FuzzyGrim

src/app/templates/app/profile.html

@@ -46,10 +46,7 @@
         <form class="p-2 grid-item" method="POST" enctype="multipart/form-data">
           {% csrf_token %}
           <div class="form-title">The Movie Database</div>
-          <div class="input-group">
-            <input type="file" accept=".csv" class="form-control" aria-label="Upload" name="tmdb">
-            <button class="btn btn-submit bg-dark rounded-end" type="submit"><i class="bi bi-check-lg"></i></button>
-          </div>
+          <button name="tmdb" class="btn btn-submit bg-dark rounded-end w-100" type="submit">TMDB Authentication</button>
         </form>
 
         <form class="p-2 grid-item" method="POST" enctype="multipart/form-data">

src/app/tests/test_imports.py

@@ -1,7 +1,7 @@
 from django.test import TestCase
 from django.test import override_settings
 
-import csv
+from unittest.mock import patch
 import shutil
 import os
 
@@ -45,71 +45,44 @@ class ImportsTMDB(TestCase):
     def setUp(self):
         self.credentials = {"username": "test", "password": "12345"}
         self.user = User.objects.create_user(**self.credentials)
-
-    @override_settings(MEDIA_ROOT=("TMDB"))
-    def test_import_tmdb(self):
         os.makedirs("TMDB")
-        file_path = os.path.join("TMDB", "ratings.csv")
 
-        fields = [
-            "TMDb ID",
-            "IMDb ID",
-            "Type",
-            "Name",
-            "Release Date",
-            "Season Number",
-            "Episode Number",
-            "Rating",
-            "Your Rating",
-            "Date Rated",
-        ]
-        data = [
-            [
-                "634649",
-                "tt10872600",
-                "movie",
-                "Spider-Man: No Way Home",
-                "2021-12-15T00:00:00Z",
-                "",
-                "",
-                "8.022",
-                "7",
-                "2022-12-17T15:50:35Z",
-            ],
-            [
-                "1668",
-                "tt0108778",
-                "tv",
-                "Friends",
-                "1994-09-22T00:00:00Z",
-                "",
-                "",
-                "8.463",
-                "10",
-                "2022-12-17T16:23:01Z",
+    @override_settings(MEDIA_ROOT=("TMDB"))
+    @patch("requests.get")
+    def test_import_tmdb(self, mock_data):
+        mock_data.return_value.json.return_value = {
+            "total_pages": 1,
+            "results": [
+                {
+                    "id": 361743,
+                    "title": "Top Gun: Maverick",
+                    "rating": 7,
+                    "poster_path": "/62HCnUTziyWcpDaBO2i1DX17ljH.jpg",
+                },
+                {
+                    "id": 634649,
+                    "title": "Spider-Man: No Way Home",
+                    "rating": 7,
+                    "poster_path": "/uJYYizSuA9Y3DCs0qS4qWvHfZg4.jpg",
+                },
             ],
-        ]
-
-        with open(file_path, "w", newline="") as f:
-            writer = csv.writer(f)
-            writer.writerow(fields)
-            writer.writerows(data)
+        }
+        fake_url = "https://api.themoviedb.org/3/account/1/rated/movies?api_key=12345&session_id=12345"
+        images, bulk_add_media = tmdb.process_media_list(
+            fake_url, "movie", "Completed", self.user, bulk_add_media=[]
+        )
+        Media.objects.bulk_create(bulk_add_media)
 
-        with open(file_path, "rb") as file:
-            tmdb.import_tmdb(file, self.user)
-            self.assertEqual(Media.objects.filter(user=self.user).count(), 2)
-            self.assertEqual(
-                Media.objects.filter(user=self.user, media_type="movie").count(), 1
-            )
-            self.assertEqual(
-                Media.objects.filter(user=self.user, media_type="tv").count(), 1
-            )
-            self.assertEqual(
-                Media.objects.get(user=self.user, media_id=634649).score == 7, True
-            )
-            self.assertEqual(
-                Media.objects.get(user=self.user, media_id=1668).score == 10, True
-            )
+        self.assertEqual(Media.objects.filter(user=self.user).count(), 2)
+        self.assertEqual(
+            Media.objects.filter(user=self.user, media_type="movie").count(), 2
+        )
+        self.assertEqual(
+            Media.objects.get(user=self.user, media_id=634649).score == 7, True
+        )
+        self.assertEqual(
+            Media.objects.get(user=self.user, media_id=361743).score == 7, True
+        )
 
     def tearDown(self):
         shutil.rmtree("TMDB")

src/app/utils/helpers.py

@@ -1,9 +1,14 @@
 from django.conf import settings
 
 import aiofiles
+import aiohttp
+import asyncio
 import datetime
 import requests
 import pathlib
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 def download_image(url, media_type):
@@ -22,6 +27,14 @@ def download_image(url, media_type):
     return filename
 
 
+async def images_downloader(images_to_download, media_type):
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for url in images_to_download:
+            tasks.append(download_image_async(session, url, media_type))
+        await asyncio.gather(*tasks)
+
+
 async def download_image_async(session, url, media_type):
     # rsplit is used to split the url at the last / and taking the last element
     # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
@@ -36,8 +49,7 @@ async def download_image_async(session, url, media_type):
                 f = await aiofiles.open(location, mode="wb")
                 await f.write(await resp.read())
                 await f.close()
-
-    return filename
+                logger.info(f"Downloaded {filename}")
 
 
 def clean_data(request, metadata):

src/app/utils/imports/anilist.py

@@ -1,6 +1,4 @@
-from aiohttp import ClientSession
-from asyncio import ensure_future, gather, run
-
+import asyncio
 import datetime
 import requests
 import logging
@@ -89,51 +87,51 @@ def import_anilist(username, user):
             return "User not found"
 
     # error stores media titles that don't have a corresponding MAL ID
-    bulk_add_media, error = run(anilist_get_media_list(query, error="", user=user))
-    Media.objects.bulk_create(bulk_add_media)
+    error = add_media_list(query, error="", user=user)
 
     logger.info(
         f"Finished importing {username} from Anilist"
     )
     return error
 
 
-async def anilist_get_media_list(query, error, user):
-    async with ClientSession() as session:
-        task = []
-        for media_type in query["data"]:
-            for list in query["data"][media_type]["lists"]:
-                if not list["isCustomList"]:
-                    for content in list["entries"]:
-                        if content["media"]["idMal"] is None:
-                            error += f"\n {content['media']['title']['userPreferred']}"
-                            logger.warning(
-                                f"{media_type.capitalize()}: {content['media']['title']['userPreferred']} has no MAL ID."
-                            )
-                        elif await Media.objects.filter(
-                            media_id=content["media"]["idMal"],
-                            media_type=media_type,
-                            user=user,
-                        ).aexists():
-                            logger.warning(
-                                f"{media_type.capitalize()}: {content['media']['title']['userPreferred']} ({content['media']['idMal']}) already exists in database. Skipping..."
-                            )
-                        else:
-                            task.append(
-                                ensure_future(
-                                    anilist_get_media(
-                                        session, content, media_type, user
-                                    )
-                                )
-                            )
-                            logger.info(
-                                f"{media_type.capitalize()}: {content['media']['title']['userPreferred']} ({content['media']['idMal']}) added to import list."
-                            )
-
-        return await gather(*task), error
-
-
-async def anilist_get_media(session, content, media_type, user):
+def add_media_list(query, error, user):
+    bulk_add_media = []
+
+    for media_type in query["data"]:
+        images_to_download = []
+        for status_list in query["data"][media_type]["lists"]:
+            if not status_list["isCustomList"]:
+                for content in status_list["entries"]:
+                    if content["media"]["idMal"] is None:
+                        error += f"\n {content['media']['title']['userPreferred']}"
+                        logger.warning(
+                            f"{media_type.capitalize()}: {content['media']['title']['userPreferred']} has no MAL ID."
+                        )
+                    elif Media.objects.filter(
+                        media_id=content["media"]["idMal"],
+                        media_type=media_type,
+                        user=user,
+                    ).exists():
+                        logger.warning(
+                            f"{media_type.capitalize()}: {content['media']['title']['userPreferred']} ({content['media']['idMal']}) already exists, skipping..."
+                        )
+                    else:
+                        images_to_download, bulk_add_media = process_media(
+                            content, media_type, user, images_to_download, bulk_add_media
+                        )
+
+                        logger.info(
+                            f"{media_type.capitalize()}: {content['media']['title']['userPreferred']} ({content['media']['idMal']}) added to import list."
+                        )
+        asyncio.run(helpers.images_downloader(images_to_download, media_type))
+
+    Media.objects.bulk_create(bulk_add_media)
+
+    return error
+
+
+def process_media(content, media_type, user, images_to_download, bulk_add_media):
     if content["status"] == "CURRENT":
         status = "Watching"
     else:
@@ -166,10 +164,13 @@ async def anilist_get_media(session, content, media_type, user):
         end_date=end_date,
     )
 
-    filename = await helpers.download_image_async(
-        session, content["media"]["coverImage"]["large"], media_type
-    )
+    bulk_add_media.append(media)
+
+    image_url = content["media"]["coverImage"]["large"]
+    images_to_download.append(image_url)
 
-    media.image = f"{filename}"
+    # rsplit is used to split the url at the last / and taking the last element
+    # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
+    media.image = f"{media_type}-{image_url.rsplit('/', 1)[-1]}"
 
-    return media
+    return images_to_download, bulk_add_media

src/app/utils/imports/mal.py

@@ -1,7 +1,6 @@
-from aiohttp import ClientSession
-from asyncio import ensure_future, gather, run
 from decouple import config
 
+import asyncio
 import datetime
 import requests
 import logging
@@ -34,6 +33,8 @@ def import_myanimelist(username, user):
         # Update the "paging" key with the new "next" URL (if any)
         animes["paging"] = next_data["paging"]
 
+    bulk_add_media = add_media_list(animes, "anime", user)
+
     manga_url = f"https://api.myanimelist.net/v2/users/{username}/mangalist?fields=list_status&nsfw=true&limit=100"
     mangas = requests.get(manga_url, headers=header).json()
 
@@ -46,43 +47,39 @@ def import_myanimelist(username, user):
         # Update the "paging" key with the new "next" URL (if any)
         mangas["paging"] = next_data["paging"]
 
-    series = {"anime": animes, "manga": mangas}
+    bulk_add_media.extend(add_media_list(mangas, "manga", user))
 
-    bulk_add_media = run(myanilist_get_media_list(series, user))
     Media.objects.bulk_create(bulk_add_media)
-
     logger.info(f"Finished importing {username} from MyAnimeList")
 
     return True
 
 
-async def myanilist_get_media_list(series, user):
-    async with ClientSession() as session:
-        task = []
-        for media_type, media_list in series.items():
-            for content in media_list["data"]:
-                if await Media.objects.filter(
-                    media_id=content["node"]["id"],
-                    media_type=media_type,
-                    user=user,
-                ).aexists():
-                    logger.warning(
-                        f"{media_type.capitalize()}: {content['node']['title']} ({content['node']['id']}) already exists in database. Skipping..."
-                    )
-                else:
-                    task.append(
-                        ensure_future(
-                            myanimelist_get_media(session, content, media_type, user)
-                        )
-                    )
-                    logger.info(
-                        f"{media_type.capitalize()}: {content['node']['title']} ({content['node']['id']}) added to import list."
-                    )
-
-        return await gather(*task)
-
-
-async def myanimelist_get_media(session, content, media_type, user):
+def add_media_list(response, media_type, user):
+    bulk_add_media = []
+    images_to_download = []
+    for content in response["data"]:
+        if Media.objects.filter(
+            media_id=content["node"]["id"],
+            media_type=media_type,
+            user=user,
+        ).exists():
+            logger.warning(
+                f"{media_type.capitalize()}: {content['node']['title']} ({content['node']['id']}) already exists, skipping..."
+            )
+        else:
+            images_to_download, bulk_add_media = process_media(content, media_type, user, images_to_download, bulk_add_media)
+
+            logger.info(
+                f"{media_type.capitalize()}: {content['node']['title']} ({content['node']['id']}) added to import list."
+            )
+
+    asyncio.run(helpers.images_downloader(images_to_download, media_type))
+
+    return bulk_add_media
+
+
+def process_media(content, media_type, user, images_to_download, bulk_add_media):
     if content["list_status"]["status"] == "plan_to_watch":
         content["list_status"]["status"] = "Planning"
     elif content["list_status"]["status"] == "on_hold":
@@ -121,12 +118,16 @@ async def myanimelist_get_media(session, content, media_type, user):
         media.end_date = None
 
     if "main_picture" in content["node"]:
-        filename = await helpers.download_image_async(
-            session, content["node"]["main_picture"]["large"], media_type
-        )
-        media.image = f"{filename}"
+        image_url = content['node']['main_picture']['large']
+        images_to_download.append(image_url)
+
+        # rsplit is used to split the url at the last / and taking the last element
+        # https://api-cdn.myanimelist.net/images/anime/12/76049.jpg -> 76049.jpg
+        media.image = f"{media_type}-{image_url.rsplit('/', 1)[-1]}"
 
     else:
         media.image = "none.svg"
 
-    return media
+    bulk_add_media.append(media)
+
+    return images_to_download, bulk_add_media