Fix #66. Optimize variants creation.

backend/spellbook/migrations/0008_remove_job_variants_variant_generated_by.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.2 on 2022-11-27 11:00
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('spellbook', '0007_alter_template_scryfall_query'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='job',
+            name='variants',
+        ),
+        migrations.AddField(
+            model_name='variant',
+            name='generated_by',
+            field=models.ForeignKey(blank=True, editable=False, help_text='Job that generated this variant', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='variants', to='spellbook.job'),
+        ),
+    ]

backend/spellbook/models.py

@@ -129,6 +129,55 @@ def ingredients(self):
         return ' + '.join([str(card) for card in self.uses.all()] + [str(feature) for feature in self.needs.all()] + [str(template) for template in self.requires.all()])
 
 
+class Job(models.Model):
+    class Status(models.TextChoices):
+        SUCCESS = 'S'
+        FAILURE = 'F'
+        PENDING = 'P'
+    name = models.CharField(max_length=255, blank=False, verbose_name='name of job')
+    created = models.DateTimeField(auto_now_add=True, blank=False)
+    expected_termination = models.DateTimeField(blank=False)
+    termination = models.DateTimeField(blank=True, null=True)
+    status = models.CharField(choices=Status.choices, default=Status.PENDING, max_length=2, blank=False)
+    message = models.TextField(blank=True)
+    started_by = models.ForeignKey(
+        to=User,
+        related_name='started_jobs',
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+        help_text='User that started this job')
+
+    def start(name: str, duration: timezone.timedelta, user: User):
+        try:
+            with transaction.atomic():
+                if Job.objects.filter(
+                        name=name,
+                        expected_termination__gte=timezone.now(),
+                        status=Job.Status.PENDING).exists():
+                    return None
+                return Job.objects.create(
+                    name=name,
+                    expected_termination=timezone.now() + duration,
+                    started_by=user)
+        except OperationalError:
+            return None
+
+    class Meta:
+        ordering = ['-created', 'name']
+        verbose_name = 'job'
+        verbose_name_plural = 'jobs'
+        indexes = [
+            models.Index(fields=['name'], name='job_name_index')
+        ]
+        constraints = [
+            models.CheckConstraint(check=models.Q(expected_termination__gte=models.F('created')), name='job_expected_termination_gte_created')
+        ]
+
+    def __str__(self):
+        return self.name
+
+
 class Variant(models.Model):
     class Status(models.TextChoices):
         NEW = 'N'
@@ -174,6 +223,7 @@ class Status(models.TextChoices):
     unique_id = models.CharField(max_length=128, unique=True, blank=False, help_text='Unique ID for this variant', editable=False)
     frozen = models.BooleanField(default=False, blank=False, help_text='Is this variant undeletable?', verbose_name='is frozen')
     identity = models.CharField(max_length=5, blank=True, help_text='Mana identity', verbose_name='mana identity', editable=False, validators=[IDENTITY_VALIDATOR])
+    generated_by = models.ForeignKey(Job, on_delete=models.SET_NULL, null=True, blank=True, editable=False, help_text='Job that generated this variant', related_name='variants')
 
     class Meta:
         ordering = ['-status', '-created']
@@ -190,60 +240,3 @@ def __str__(self):
         return ' + '.join([str(card) for card in self.uses.all()] + [str(template) for template in self.requires.all()]) \
             + ' ➡ ' + ' + '.join([str(feature) for feature in produces[:3]]) \
             + ('...' if len(produces) > 3 else '')
-
-
-class Job(models.Model):
-    class Status(models.TextChoices):
-        SUCCESS = 'S'
-        FAILURE = 'F'
-        PENDING = 'P'
-    name = models.CharField(max_length=255, blank=False, verbose_name='name of job')
-    created = models.DateTimeField(auto_now_add=True, blank=False)
-    expected_termination = models.DateTimeField(blank=False)
-    termination = models.DateTimeField(blank=True, null=True)
-    status = models.CharField(choices=Status.choices, default=Status.PENDING, max_length=2, blank=False)
-    message = models.TextField(blank=True)
-    started_by = models.ForeignKey(
-        to=User,
-        related_name='started_jobs',
-        blank=True,
-        null=True,
-        on_delete=models.SET_NULL,
-        help_text='User that started this job')
-    variants = models.ManyToManyField(
-        to=Variant,
-        related_name='jobs',
-        blank=True,
-        help_text='Variants that this job added or updated',
-        verbose_name='variants updated',
-        editable=False
-    )
-
-    def start(name: str, duration: timezone.timedelta, user: User):
-        try:
-            with transaction.atomic():
-                if Job.objects.filter(
-                        name=name,
-                        expected_termination__gte=timezone.now(),
-                        status=Job.Status.PENDING).exists():
-                    return None
-                return Job.objects.create(
-                    name=name,
-                    expected_termination=timezone.now() + duration,
-                    started_by=user)
-        except OperationalError:
-            return None
-
-    class Meta:
-        ordering = ['-created', 'name']
-        verbose_name = 'job'
-        verbose_name_plural = 'jobs'
-        indexes = [
-            models.Index(fields=['name'], name='job_name_index')
-        ]
-        constraints = [
-            models.CheckConstraint(check=models.Q(expected_termination__gte=models.F('created')), name='job_expected_termination_gte_created')
-        ]
-
-    def __str__(self):
-        return self.name

backend/spellbook/variants/variant_data.py

@@ -33,6 +33,8 @@ def __init__(self):
         self.not_working_variants = fetch_not_working_variants(self.variants)
         self.uid_to_variant = {v.unique_id: v for v in self.variants}
         self.combo_to_removed_features = fetch_removed_features(self.combos)
+        self.id_to_combo = {c.id: c for c in self.combos}
+        self.id_to_card = {c.id: c for c in self.cards}
 
 
 count = 0

backend/spellbook/variants/variants_generator.py

@@ -86,8 +86,9 @@ def update_variant(
 def create_variant(
         data: Data,
         unique_id: str,
-        variant_def: VariantDefinition):
-    combos = data.combos.filter(id__in=variant_def.included_ids)
+        variant_def: VariantDefinition,
+        job: Job = None):
+    combos = [data.id_to_combo[c_id] for c_id in variant_def.included_ids]
     zone_locations = '\n'.join(c.zone_locations for c in combos if len(c.zone_locations) > 0)
     cards_state = '\n'.join(c.cards_state for c in combos if len(c.cards_state) > 0)
     other_prerequisites = '\n'.join(c.other_prerequisites for c in combos if len(c.other_prerequisites) > 0)
@@ -101,7 +102,8 @@ def create_variant(
         other_prerequisites=other_prerequisites,
         mana_needed=mana_needed,
         description=description,
-        identity=merge_identities(data.cards.filter(id__in=variant_def.card_ids).values_list('identity', flat=True)))
+        identity=merge_identities([data.id_to_card[c_id].identity for c_id in variant_def.card_ids]),
+        generated_by=job)
     if not ok:
         variant.status = Variant.Status.NOT_WORKING
     return VariantBulkSaveItem(
@@ -153,7 +155,6 @@ def get_variants_from_graph(data: Data, job: Job = None) -> dict[str, VariantDef
 
 def perform_bulk_saves(to_create: list[VariantBulkSaveItem], to_update: list[VariantBulkSaveItem]):
     batch_size = 999
-    debug_queries(True)
     Variant.objects.bulk_create((v.variant for v in to_create if v.should_save), batch_size=batch_size)
     update_fields = ['identity', 'zone_locations', 'cards_state', 'mana_needed', 'other_prerequisites', 'description', 'status']
     Variant.objects.bulk_update((v.variant for v in to_update if v.should_save), fields=update_fields, batch_size=batch_size)
@@ -170,7 +171,6 @@ def perform_bulk_saves(to_create: list[VariantBulkSaveItem], to_update: list[Var
     ProducesTable = Variant.produces.through
     ProducesTable.objects.all().delete()
     ProducesTable.objects.bulk_create((ProducesTable(variant_id=v.variant.id, feature_id=f) for v in to_create + to_update for f in v.produces), batch_size=batch_size)
-    debug_queries(True)
 
 
 def generate_variants(job: Job = None) -> tuple[int, int, int]:
@@ -203,12 +203,10 @@ def generate_variants(job: Job = None) -> tuple[int, int, int]:
                 variant_to_save = create_variant(
                     data=data,
                     unique_id=unique_id,
-                    variant_def=variant_def)
+                    variant_def=variant_def,
+                    job=job)
                 to_bulk_create.append(variant_to_save)
         perform_bulk_saves(to_bulk_create, to_bulk_update)
-        if job is not None:
-            # TODO set which variants were generated for this job
-            pass
         new_id_set = set(variants.keys())
         to_delete = old_id_set - new_id_set
         added = new_id_set - old_id_set