Export model data to JS instead of letting the JS compilation query t…

…he database (#9674)

* Add Frontend JS export routine

* Use stored JSON data in frontend compilation

* Add Rake task to generate frontend files during Docker spinup

* Add data files in raw JSON form

* Refactor into abstract data loader

* Add migration to remove primary key from championship ISO2 table

* Add basic StaticData module for easy loading inclusion

* Add backend loader model

* Adjust frontend data loader to use exported file format

* Replace database call on Competition.non_future_years

* Fix DB access in EligibleIso2 validator

* Run frontend data export in GitHub CI

* Consistently use camelCase in data loading

* Share derived property computation between JS and Rails

* Remove frontend compilation task

* Redirect model listener to lib/static_files folder

* Fix data compilation errors

* Remove direct AR instantiations from model code

* Symbolize keys upon static_file JSON parsing

* Fix 'official' computation of Events in frontend

* Let static data handle their own export

* Remove static data listener

* Dump only fictive countries for Country static_data

* Don't deliver duplicated data files through Webpacker

* Regenerate data files after backend refactor

* Add custom PreferredFormat dumper

* Rename loader method

* Fix column sanitizing for static data

* Remove cellName from Events table

* Fix upsert call in old migrations

* Add isOfficial to Events serialization format

* Hard-code competition model length validations

* Add rake task for handling static data load/dump

* Fix dump/load consistency in events.json

* Fix classloader cyclic dependency

* Avoid constant initialization DB access in SolveTime

* Use compact export format for PreferredFormat

* Remove legacy test case for 333ft

app/controllers/results_controller.rb

@@ -170,7 +170,6 @@ def records
           DAY(competition.start_date)   day,
           event.id             eventId,
           event.name           eventName,
-          event.cellName       eventCellName,
           result.type          type,
           result.value         value,
           result.formatId      formatId,
@@ -224,7 +223,6 @@ def records
                              result.*,
                              value,
         event.name           eventName,
-        event.cellName       eventCellName,
                              format,
         country.name         countryName,
         competition.cellName competitionName,

app/models/championship.rb

@@ -5,11 +5,11 @@ class Championship < ApplicationRecord
   CHAMPIONSHIP_TYPE_WORLD = "world"
   MAJOR_CHAMPIONSHIP_TYPES = [
     CHAMPIONSHIP_TYPE_WORLD,
-    *Continent.real.map(&:id),
+    *Continent::REAL_CONTINENTS.pluck(:id),
   ].freeze
   CHAMPIONSHIP_TYPES = [
     *MAJOR_CHAMPIONSHIP_TYPES,
-    *Country.real.map(&:iso2),
+    *Country::WCA_COUNTRIES.pluck(:iso2),
     *EligibleCountryIso2ForChampionship.championship_types,
   ].freeze
 
@@ -58,13 +58,4 @@ def to_a
   def <=>(other)
     self.to_a <=> other.to_a
   end
-
-  def self.grouped_championship_types
-    {
-      "planetary" => [CHAMPIONSHIP_TYPE_WORLD],
-      "continental" => Continent.all_sorted_by(I18n.locale, real: true).map(&:id),
-      "multi-national" => EligibleCountryIso2ForChampionship.championship_types,
-      "national" => Country.all_sorted_by(I18n.locale, real: true).map(&:iso2),
-    }
-  end
 end

app/models/competition.rb

@@ -176,6 +176,11 @@ class Competition < ApplicationRecord
   MAX_ID_LENGTH = 32
   MAX_NAME_LENGTH = 50
   MAX_CELL_NAME_LENGTH = 32
+  MAX_CITY_NAME_LENGTH = 50
+  MAX_VENUE_LENGTH = 240
+  MAX_FREETEXT_LENGTH = 191
+  MAX_URL_LENGTH = 200
+  MAX_MARKDOWN_LENGTH = 255
   MAX_COMPETITOR_LIMIT = 5000
   MAX_GUEST_LIMIT = 100
   validates_inclusion_of :competitor_limit_enabled, in: [true, false], if: :competitor_limit_required?
@@ -197,7 +202,7 @@ class Competition < ApplicationRecord
   validates :external_website, format: { with: URL_RE }, allow_blank: true
   validates :external_registration_page, presence: true, format: { with: URL_RE }, if: :external_registration_page_required?
 
-  validates_inclusion_of :countryId, in: Country.ids.freeze
+  validates_inclusion_of :countryId, in: Country::ALL_STATES_RAW.pluck(:id).freeze
   validates :currency_code, inclusion: { in: Money::Currency, message: proc { I18n.t('competitions.errors.invalid_currency_code') } }
 
   validates_numericality_of :refund_policy_percent, greater_than_or_equal_to: 0, less_than_or_equal_to: 100, if: :refund_policy_percent_required?
@@ -224,17 +229,13 @@ class Competition < ApplicationRecord
   validates_inclusion_of :main_event_id, in: ->(comp) { [nil].concat(comp.persisted_events_id) }
 
   # Validations are used to show form errors to the user. If string columns aren't validated for length, it produces an unexplained error for the user
-  # VALIDATED_COLUMNS: All columns which appear in the competition form and are editable by users
-  # DONT_VALIDATE_STRING_LENGTH: String columns not exposed to users in the cmopetition form
-  VALIDATE_STRING_LENGTH = %w[
-    name cityName venue venueAddress venueDetails external_website cellName contact name_reason external_registration_page forbid_newcomers_reason
-  ].freeze
-  DONT_VALIDATE_STRING_LENGTH = %w[countryId connected_stripe_account_id currency_code main_event_id id].freeze
-  columns_hash.each do |column_name, column_info|
-    if VALIDATE_STRING_LENGTH.include?(column_name) && column_info.limit
-      validates column_name, length: { maximum: column_info.limit }
-    end
-  end
+  validates :name, length: { maximum: MAX_NAME_LENGTH }
+  validates :cellName, length: { maximum: MAX_CELL_NAME_LENGTH }
+  validates :cityName, length: { maximum: MAX_CITY_NAME_LENGTH }
+  validates :venue, length: { maximum: MAX_VENUE_LENGTH }
+  validates :venueAddress, :venueDetails, :name_reason, :forbid_newcomers_reason, length: { maximum: MAX_FREETEXT_LENGTH }
+  validates :external_website, :external_registration_page, length: { maximum: MAX_URL_LENGTH }
+  validates :contact, length: { maximum: MAX_MARKDOWN_LENGTH }
 
   # Dirty old trick to deal with competition id changes (see other methods using
   # 'with_old_id' for more details).

app/models/concerns/static_data.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module StaticData
+  extend ActiveSupport::Concern
+
+  DATA_FOLDER = Rails.root.join('lib/static_data')
+
+  class_methods do
+    def parse_json_file(file_path, symbolize_names: true)
+      ::JSON.parse(File.read(file_path), symbolize_names: symbolize_names)
+    end
+
+    def write_json_file(file_path, hash_data)
+      json_output = ::JSON.pretty_generate(hash_data.as_json)
+
+      # Don't rewrite the file if it already exists and has the same content.
+      # It helps the asset pipeline or webpack understand that file wasn't changed.
+      if File.exist?(file_path) && File.read(file_path) == json_output
+        return
+      end
+
+      File.write(file_path, json_output)
+    end
+  end
+
+  included do
+    after_commit :write_json_data! if Rails.env.development?
+
+    def self.data_file_handle
+      self.name.pluralize.underscore
+    end
+
+    def self.data_file_path
+      DATA_FOLDER.join("#{self.data_file_handle}.json")
+    end
+
+    def self.static_json_data
+      self.parse_json_file(self.data_file_path)
+    end
+
+    def self.all_raw
+      self.static_json_data
+    end
+
+    def self.all_raw_sanitized
+      column_symbols = self.column_names.map(&:to_sym)
+      self.all_raw.map { |attributes| attributes.slice(*column_symbols) }
+    end
+
+    def self.all_static
+      self.all_raw_sanitized.map { |attributes| self.new(**attributes) }
+    end
+
+    def self.dump_static
+      self.all.as_json
+    end
+
+    def self.load_json_data!
+      self.upsert_all(self.all_raw_sanitized)
+    end
+
+    def self.write_json_data!
+      self.write_json_file(self.data_file_path, self.dump_static)
+    end
+  end
+end

app/models/continent.rb

@@ -7,6 +7,9 @@ class Continent < ApplicationRecord
 
   include Cachable
   include LocalizedSortable
+  include StaticData
+
+  REAL_CONTINENTS = self.all_raw.select { |c| !FICTIVE_IDS.include?(c[:id]) }.freeze
 
   has_many :countries, foreign_key: :continentId
 

app/models/country.rb

@@ -2,7 +2,8 @@
 
 class Country < ApplicationRecord
   include Cachable
-  WCA_STATES_JSON_PATH = Rails.root.to_s + "/config/wca-states.json"
+  include StaticData
+
   self.table_name = "Countries"
 
   has_one :wfc_dues_redirect, as: :redirect_source
@@ -27,33 +28,45 @@ class Country < ApplicationRecord
     all_tz
   end.freeze
 
-  MULTIPLE_COUNTRIES = [
-    { id: 'XF', name: 'Multiple Countries (Africa)', continentId: '_Africa', iso2: 'XF' },
-    { id: 'XM', name: 'Multiple Countries (Americas)', continentId: '_Multiple Continents', iso2: 'XM' },
-    { id: 'XA', name: 'Multiple Countries (Asia)', continentId: '_Asia', iso2: 'XA' },
-    { id: 'XE', name: 'Multiple Countries (Europe)', continentId: '_Europe', iso2: 'XE' },
-    { id: 'XN', name: 'Multiple Countries (North America)', continentId: '_North America', iso2: 'XN' },
-    { id: 'XO', name: 'Multiple Countries (Oceania)', continentId: '_Oceania', iso2: 'XO' },
-    { id: 'XS', name: 'Multiple Countries (South America)', continentId: '_South America', iso2: 'XS' },
-    { id: 'XW', name: 'Multiple Countries (World)', continentId: '_Multiple Continents', iso2: 'XW' },
-  ].freeze
-
-  FICTIVE_IDS = MULTIPLE_COUNTRIES.map { |c| c[:id] }.freeze
+  FICTIVE_COUNTRY_DATA_PATH = StaticData::DATA_FOLDER.join("#{self.data_file_handle}.fictive.json")
+  MULTIPLE_COUNTRIES = self.parse_json_file(FICTIVE_COUNTRY_DATA_PATH).freeze
+
+  FICTIVE_IDS = MULTIPLE_COUNTRIES.pluck(:id).freeze
   NAME_LOOKUP_ATTRIBUTE = :iso2
+
   include LocalizedSortable
 
-  WCA_STATES = JSON.parse(File.read(WCA_STATES_JSON_PATH)).freeze
+  REAL_COUNTRY_DATA_PATH = StaticData::DATA_FOLDER.join("#{self.data_file_handle}.real.json")
+  WCA_STATES_JSON = self.parse_json_file(REAL_COUNTRY_DATA_PATH, symbolize_names: false).freeze
 
-  ALL_STATES = [
-    WCA_STATES["states_lists"].map do |list|
-      list["states"].map do |state|
-        state_id = state["id"] || I18n.transliterate(state["name"]).tr("'", "_")
-        { id: state_id, continentId: state["continent_id"],
-          iso2: state["iso2"], name: state["name"] }
-      end
-    end,
+  WCA_COUNTRIES = WCA_STATES_JSON["states_lists"].flat_map do |list|
+    list["states"].map do |state|
+      state_id = state["id"] || I18n.transliterate(state["name"]).tr("'", "_")
+      { id: state_id, continentId: state["continent_id"],
+        iso2: state["iso2"], name: state["name"] }
+    end
+  end
+
+  ALL_STATES_RAW = [
+    WCA_COUNTRIES,
     MULTIPLE_COUNTRIES,
-  ].flatten.map { |c| Country.new(c) }.freeze
+  ].flatten.freeze
+
+  def self.all_raw
+    ALL_STATES_RAW
+  end
+
+  # As of writing this comment, the actual `Countries` data is controlled by WRC
+  # and we only have control over the 'fictive' values. We parse the WRC file above and override
+  # the `all_raw` getter to include the real countries, but they're not part of our static dataset in the stricter sense
+
+  def self.dump_static
+    MULTIPLE_COUNTRIES
+  end
+
+  def self.data_file_handle
+    "#{self.name.pluralize.underscore}.fictive"
+  end
 
   belongs_to :continent, foreign_key: :continentId
   alias_attribute :continent_id, :continentId

app/models/eligible_country_iso2_for_championship.rb

@@ -1,14 +1,35 @@
 # frozen_string_literal: true
 
 class EligibleCountryIso2ForChampionship < ApplicationRecord
+  include StaticData
+
   self.table_name = "eligible_country_iso2s_for_championship"
 
   belongs_to :championship, foreign_key: :championship_type, primary_key: :championship_type, optional: true
 
   validates :eligible_country_iso2, uniqueness: { scope: :championship_type, case_sensitive: false },
-                                    inclusion: { in: Country.all.map(&:iso2) }
+                                    inclusion: { in: Country::ALL_STATES_RAW.pluck(:iso2) }
+
+  def self.data_file_handle
+    "championship_eligible_iso2"
+  end
+
+  def self.all_raw
+    self.static_json_data.flat_map do |type, iso2_list|
+      iso2_list.map do |iso2|
+        { championship_type: type, eligible_country_iso2: iso2 }
+      end
+    end
+  end
+
+  def self.dump_static
+    self.all
+        .group_by(&:championship_type)
+        .transform_values { |el| el.pluck(:eligible_country_iso2) }
+        .as_json
+  end
 
   def self.championship_types
-    pluck(:championship_type).uniq
+    all_raw.pluck(:championship_type).uniq
   end
 end

app/models/event.rb

@@ -2,6 +2,8 @@
 
 class Event < ApplicationRecord
   include Cachable
+  include StaticData
+
   self.table_name = "Events"
 
   has_many :competition_events
@@ -23,10 +25,6 @@ def name_in(locale)
     I18n.t(id, scope: :events, locale: locale)
   end
 
-  def cellName
-    raise "#cellName is deprecated, and will eventually be removed. Use #name instead. See https://github.com/thewca/worldcubeassociation.org/issues/1054."
-  end
-
   # Pay special attention to the difference between .. (two dots) and ... (three dots)
   # which map to different operators < and <= in SQL (inclusive VS exclusive range)
   scope :official, -> { where(rank: ...990) }
@@ -70,16 +68,23 @@ def fast_event?
     ['333', '222', '444', '333oh', 'clock', 'mega', 'pyram', 'skewb', 'sq1'].include?(self.id)
   end
 
+  def self.dump_static
+    self.includes(:preferred_formats, :formats).order(:rank).as_json(
+      only: %w[id rank format],
+    )
+  end
+
   alias_method :can_change_time_limit, :can_change_time_limit?
   alias_method :can_have_cutoff, :can_have_cutoff?
   alias_method :is_timed_event, :timed_event?
   alias_method :is_fewest_moves, :fewest_moves?
   alias_method :is_multiple_blindfolded, :multiple_blindfolded?
+  alias_method :is_official, :official?
 
   DEFAULT_SERIALIZE_OPTIONS = {
     only: ["id"],
     methods: ["name", "can_change_time_limit", "can_have_cutoff", "is_timed_event",
-              "is_fewest_moves", "is_multiple_blindfolded", "format_ids"],
+              "is_fewest_moves", "is_multiple_blindfolded", "is_official", "format_ids"],
   }.freeze
 
   def serializable_hash(options = nil)

app/models/format.rb

@@ -2,6 +2,8 @@
 
 class Format < ApplicationRecord
   include Cachable
+  include StaticData
+
   self.table_name = "Formats"
 
   has_many :preferred_formats

app/models/preferred_format.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class PreferredFormat < ApplicationRecord
+  include StaticData
+
   belongs_to :event
   belongs_to :format
 
@@ -9,4 +11,21 @@ def format
   end
 
   default_scope -> { order(:ranking) }
+
+  def self.all_raw
+    self.static_json_data.flat_map do |event_id, format_ids|
+      format_ids.map.with_index do |format_id, idx|
+        { event_id: event_id, format_id: format_id, ranking: idx + 1 }
+      end
+    end
+  end
+
+  def self.dump_static
+    self.unscoped
+        .joins(:event)
+        .order(:rank)
+        .group_by(&:event_id)
+        .transform_values { |el| el.sort_by(&:ranking).pluck(:format_id) }
+        .as_json
+  end
 end

app/models/round_type.rb

@@ -2,6 +2,8 @@
 
 class RoundType < ApplicationRecord
   include Cachable
+  include StaticData
+
   self.table_name = "RoundTypes"
 
   has_many :results, foreign_key: :roundTypeId

app/views/regulations/countries.html.erb

@@ -1,8 +1,8 @@
-<% title = Country::WCA_STATES["title"] %>
-<% version = Country::WCA_STATES["version"] %>
-<% version_hash = Country::WCA_STATES["version_hash"] %>
-<% preamble = Country::WCA_STATES["text"] %>
-<% states_lists = Country::WCA_STATES["states_lists"] %>
+<% title = Country::WCA_STATES_JSON["title"] %>
+<% version = Country::WCA_STATES_JSON["version"] %>
+<% version_hash = Country::WCA_STATES_JSON["version_hash"] %>
+<% preamble = Country::WCA_STATES_JSON["text"] %>
+<% states_lists = Country::WCA_STATES_JSON["states_lists"] %>
 <% provide(:title, title) %>
 <div class="container">
   <h1><%= title %></h1>