From 1240eac9668fedfe5086f2feebe0cb79a4dd7bb4 Mon Sep 17 00:00:00 2001 From: Finomnis Date: Sun, 12 Nov 2023 13:56:41 +0100 Subject: [PATCH] Rework font generator (#19) * Parallelize font creation * Add progress bar * Make progress bar hidable --- .github/workflows/ci.yml | 2 +- tools/generate_fonts_file/Cargo.lock | 230 ++++++++++++++++++-- tools/generate_fonts_file/Cargo.toml | 2 + tools/generate_fonts_file/src/font_data.rs | 6 +- tools/generate_fonts_file/src/font_entry.rs | 14 +- tools/generate_fonts_file/src/main.rs | 88 +++++--- 6 files changed, 288 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdd030a..170a9ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,7 +214,7 @@ jobs: #- uses: Swatinem/rust-cache@v1 - name: Regenerate and check - run: ./regenerate_fonts_file.sh --check + run: ./regenerate_fonts_file.sh --check --hide-progress release: name: Publish version diff --git a/tools/generate_fonts_file/Cargo.lock b/tools/generate_fonts_file/Cargo.lock index b481113..1fb0617 100644 --- a/tools/generate_fonts_file/Cargo.lock +++ b/tools/generate_fonts_file/Cargo.lock @@ -62,7 +62,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -72,9 +72,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + [[package]] name = "backtrace" version = "0.3.66" @@ -165,6 +171,64 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "errno" version = "0.3.1" @@ -173,7 +237,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -191,8 +255,10 @@ name = "generate_fonts_file" version = "0.1.0" dependencies = [ "clap", + "indicatif", "lazy_static", "miette", + "rayon", "regex", ] @@ -214,6 +280,29 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "rayon", + "unicode-width", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -222,7 +311,7 @@ checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -234,7 +323,7 @@ dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -267,6 +356,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "miette" version = "5.9.0" @@ -308,6 +406,12 @@ dependencies = [ "adler", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.29.0" @@ -329,6 +433,12 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + [[package]] name = "proc-macro2" version = "1.0.60" @@ -347,6 +457,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.8.4" @@ -381,9 +511,15 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.48.0", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "smawk" version = "0.3.1" @@ -525,13 +661,37 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -540,51 +700,93 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" diff --git a/tools/generate_fonts_file/Cargo.toml b/tools/generate_fonts_file/Cargo.toml index 969731b..e13052a 100644 --- a/tools/generate_fonts_file/Cargo.toml +++ b/tools/generate_fonts_file/Cargo.toml @@ -10,3 +10,5 @@ clap = { version = "4.3.2", features = ["derive"] } miette = { version = "5.9.0", features = ["fancy"] } regex = "1.8.4" lazy_static = "1.4.0" +rayon = "1.8.0" +indicatif = { version = "0.17.7", features = ["rayon"] } diff --git a/tools/generate_fonts_file/src/font_data.rs b/tools/generate_fonts_file/src/font_data.rs index 6636d99..36f5c75 100644 --- a/tools/generate_fonts_file/src/font_data.rs +++ b/tools/generate_fonts_file/src/font_data.rs @@ -18,7 +18,7 @@ enum Action { Finished, } -pub fn consume_font_data<'a>(mut data: &'a [u8], out: &mut Vec) -> Result<(&'a [u8], usize)> { +pub fn consume_font_data<'a>(mut data: &'a [u8], out: &mut Vec) -> Result { let mut state = State::Padding; let mut num_produced = 0; @@ -81,12 +81,12 @@ pub fn consume_font_data<'a>(mut data: &'a [u8], out: &mut Vec) -> Result<(& state = new_state; } Action::Finished => { - data = &data[1..]; + //data = &data[1..]; break; } } } produce(0); - Ok((data, num_produced)) + Ok(num_produced) } diff --git a/tools/generate_fonts_file/src/font_entry.rs b/tools/generate_fonts_file/src/font_entry.rs index a052f59..9205a19 100644 --- a/tools/generate_fonts_file/src/font_entry.rs +++ b/tools/generate_fonts_file/src/font_entry.rs @@ -3,12 +3,13 @@ use miette::{IntoDiagnostic, Result, WrapErr}; use regex::bytes::Regex; pub struct FontEntry<'a> { - pub name: &'a [u8], + pub data: &'a [u8], + pub name: &'a str, pub expected_length: usize, } lazy_static! { - static ref FONT_REGEX: Regex = + pub static ref FONT_REGEX: Regex = Regex::new(r#"const uint8_t (\w*)\[(\d*)\] U8G2_FONT_SECTION\("(\w*)"\) ="#,).unwrap(); } @@ -19,7 +20,7 @@ impl<'a> FontEntry<'a> { None => return Ok((data, None)), }; - let name = font_match.get(1).unwrap().as_bytes(); + let name = std::str::from_utf8(font_match.get(1).unwrap().as_bytes()).into_diagnostic()?; let expected_length: usize = String::from_utf8(font_match.get(2).unwrap().as_bytes().to_vec()) .into_diagnostic() @@ -29,15 +30,16 @@ impl<'a> FontEntry<'a> { .wrap_err("Unable to read font length")?; let name2 = font_match.get(3).unwrap().as_bytes(); - assert!(name == name2); + assert!(name.as_bytes() == name2); + + let leftover_data = &data[font_match.get(0).unwrap().range().end..]; let font_entry = FontEntry { + data: leftover_data, name, expected_length, }; - let leftover_data = &data[font_match.get(0).unwrap().range().end..]; - Ok((leftover_data, Some(font_entry))) } } diff --git a/tools/generate_fonts_file/src/main.rs b/tools/generate_fonts_file/src/main.rs index fe802c1..996f594 100644 --- a/tools/generate_fonts_file/src/main.rs +++ b/tools/generate_fonts_file/src/main.rs @@ -8,7 +8,9 @@ use std::{ }; use clap::Parser; +use indicatif::{ParallelProgressIterator, ProgressBar}; use miette::{bail, IntoDiagnostic, Result, WrapErr}; +use rayon::prelude::*; use crate::{font_data::consume_font_data, font_entry::FontEntry}; @@ -29,6 +31,10 @@ struct Args { /// differs from the existing code #[clap(long)] check: bool, + + /// Hides the progress bar + #[clap(long)] + hide_progress: bool, } fn read_input_file(file: &str) -> Result> { @@ -70,26 +76,14 @@ fn check_output_file(file: &str, data: &[u8]) -> Result<()> { Ok(()) } -fn process_font_entry<'a>( - font_entry: &FontEntry, - out: &mut Vec, - mut leftover_data: &'a [u8], -) -> Result<&'a [u8]> { - println!( - "{:>5} kB - {}", - font_entry.expected_length / 1024 + 1, - String::from_utf8(font_entry.name.to_vec()).unwrap(), - ); - +fn process_font_entry<'a>(font_entry: FontEntry<'a>, out: &mut Vec) -> Result<()> { out.extend_from_slice(b"\npub struct "); - out.extend_from_slice(font_entry.name); + out.extend_from_slice(font_entry.name.as_bytes()); out.extend_from_slice(b";\nimpl Font for "); - out.extend_from_slice(font_entry.name); + out.extend_from_slice(font_entry.name.as_bytes()); out.extend_from_slice(b" {\n const DATA: &'static [u8] = b\""); - let (d, length) = - consume_font_data(leftover_data, out).wrap_err("Unable to consume font data")?; - leftover_data = d; + let length = consume_font_data(font_entry.data, out).wrap_err("Unable to consume font data")?; miette::ensure!( length == font_entry.expected_length, @@ -100,7 +94,23 @@ fn process_font_entry<'a>( out.extend_from_slice(b"\";\n}\n"); - Ok(leftover_data) + Ok(()) +} + +fn pre_parse_fonts(mut data: &[u8]) -> Result> { + let mut result = Vec::new(); + loop { + let (leftover, entry) = FontEntry::try_consume(data)?; + data = leftover; + + match entry { + Some(entry) => { + result.push(entry); + } + None => break, + } + } + Ok(result) } fn main() -> Result<()> { @@ -112,20 +122,38 @@ fn main() -> Result<()> { out.extend_from_slice(b"use crate::Font;\n"); - let mut leftover_data = input_data.as_slice(); - loop { - let (s, font_entry) = FontEntry::try_consume(leftover_data) - .wrap_err("Error while searching for next font entry")?; - leftover_data = s; + let fonts = pre_parse_fonts(&input_data) + .wrap_err("Unable to parse fonts file!")? + .into_par_iter(); + println!("Found {} fonts.", fonts.len()); - match font_entry { - None => break, - Some(font_entry) => { - leftover_data = process_font_entry(&font_entry, &mut out, &mut leftover_data) - .wrap_err("Error while processing font entry")?; - } - } - } + let mut print_msg: Box = Box::new(|s| println!("{}", s)); + let fonts = if args.hide_progress { + fonts.progress_with(ProgressBar::hidden()) + } else { + let fonts = fonts.progress(); + let progress_bar = fonts.progress.clone(); + print_msg = Box::new(move |s| progress_bar.println(s)); + fonts + }; + + let font_data = fonts + .map(|font_entry| { + print_msg(format!( + "{:>5} kB - {}", + font_entry.expected_length / 1024 + 1, + font_entry.name, + )); + + let mut font_out = Vec::new(); + process_font_entry(font_entry, &mut font_out) + .wrap_err("Error while processing font entry") + .unwrap(); + font_out + }) + .flatten(); + + out = out.into_par_iter().chain(font_data).collect(); if args.check { check_output_file(&args.file_out, &out)