dynamic/static embeds

build/Cargo.toml

@@ -10,25 +10,8 @@ path = "src/bundle_script.rs"
 name = "script"
 
 [dependencies]
+demostf-build-derive = {version = "0.1", path = "./derive"}
+demostf-build-bundlers = {version = "0.1", path = "./bundlers"}
+rand = "0.8.5"
 tracing-subscriber = "0.3.16"
-lightningcss = { version = "1.0.0-alpha.40", features = ["browserslist", "visitor"] }
-base64 = "0.21.0"
-urlencoding = "2.1.2"
-const-fnv1a-hash = "1.1.0"
-swc = "0.259.6"
-swc_common = { version = "0.30.5", features = ["tty-emitter", "concurrent"], path = "../../../rust/swc/crates/swc_common" }
-#swc_bundler = "0.212.5"
-swc_bundler = { version = "0.212.5", path = "../../../rust/swc/crates/swc_bundler" }
-#swc_ecma_loader = "0.42.5"
-swc_ecma_loader = { version = "0.42.5", path = "../../../rust/swc/crates/swc_ecma_loader", features = ["node", "cache"] }
-swc_ecma_ast = { version = "0.102.5", path = "../../../rust/swc/crates/swc_ecma_ast" }
-swc_atoms = { version = "0.4.43", path = "../../../rust/swc/crates/swc_atoms" }
-swc_ecma_parser = { version = "0.132.6", features = ["typescript"], path = "../../../rust/swc/crates/swc_ecma_parser" }
-swc_ecma_codegen = { version = "0.137.6", path = "../../../rust/swc/crates/swc_ecma_codegen" }
-swc_ecma_transforms_base = { version = "0.125.1", path = "../../../rust/swc/crates/swc_ecma_transforms_base" }
-swc_ecma_transforms_typescript = { version = "0.175.4", path = "../../../rust/swc/crates/swc_ecma_transforms_typescript" }
-swc_ecma_visit = { version = "0.88.5", path = "../../../rust/swc/crates/swc_ecma_visit" }
-anyhow = "1.0.70"
-#jsx-dom-expressions = { version = "0.1", git = "https://github.com/icewind1991/swc-plugin-jsx-dom-expressions" }
-jsx-dom-expressions = { version = "0.1", path = "../../../rust/swc-plugin-jsx-dom-expressions" }
 

build/bundlers/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "demostf-build-bundlers"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+lightningcss = { version = "1.0.0-alpha.40", features = ["browserslist", "visitor"] }
+base64 = "0.21.0"
+urlencoding = "2.1.2"
+swc = "0.259.6"
+swc_common = { version = "0.30.5", features = ["tty-emitter", "concurrent"], git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_core = { version = "0.74.6", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+#swc_bundler = "0.212.5"
+swc_bundler = { version = "0.212.5", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+#swc_ecma_loader = "0.42.5"
+swc_ecma_loader = { version = "0.42.5", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old", features = ["node", "cache"] }
+swc_ecma_ast = { version = "0.102.5", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_atoms = { version = "0.4.43", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_ecma_parser = { version = "0.132.6", features = ["typescript"], git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_ecma_codegen = { version = "0.137.6", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_ecma_transforms_base = { version = "0.125.1", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_ecma_transforms_typescript = { version = "0.175.4", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+swc_ecma_visit = { version = "0.88.5", git = "https://github.com/icewind1991/swc", branch = "loader-browser-overwrite-old" }
+anyhow = "1.0.70"
+jsx-dom-expressions = { version = "0.1", git = "https://github.com/icewind1991/swc-plugin-jsx-dom-expressions", branch = "loader-browser-overwrite-old" }
+#jsx-dom-expressions = { version = "0.1", path = "../../../../rust/swc-plugin-jsx-dom-expressions" }
+rand = "0.8.5"
+fnv = "1.0.7"
+

build/bundlers/src/lib.rs

@@ -0,0 +1,39 @@
+mod script;
+mod style;
+
+use fnv::FnvHasher;
+pub use script::bundle_script;
+use std::fs::read;
+use std::hash::{Hash, Hasher};
+pub use style::bundle_style;
+
+pub fn bundle_raw(input: &str) -> Vec<u8> {
+    read(input).expect("failed to read raw")
+}
+
+fn guess_embed(path: &str) -> (&'static str, bool) {
+    match path.split('.').last().unwrap() {
+        "svg" => ("image/svg+xml", false),
+        "png" => ("image/png", true),
+        ext => panic!("no mimetype known for {ext}"),
+    }
+}
+
+pub fn hash(input: &[u8]) -> String {
+    let mut hasher = FnvHasher::default();
+    input.hash(&mut hasher);
+    format!("{:x}", hasher.finish())
+}
+
+pub fn guess_mime(path: &str) -> &'static str {
+    if path.ends_with("svg") {
+        return "image/svg+xml";
+    } else if path.ends_with("png") {
+        return "image/png";
+    } else if path.ends_with("css") {
+        return "text/css";
+    } else if path.ends_with("js") {
+        return "text/javascript";
+    }
+    return "text/plain";
+}

build/bundlers/src/script.rs

@@ -23,13 +23,13 @@ use swc_ecma_transforms_base::hygiene::hygiene;
 use swc_ecma_transforms_typescript::strip;
 use swc_ecma_visit::{as_folder, FoldWith};
 
-pub fn bundle_script(script: &str) -> String {
+pub fn bundle_script(script: &str) -> Vec<u8> {
     #[cfg(debug_assertions)]
     let minify = false;
     #[cfg(not(debug_assertions))]
     let minify = true;
 
-    let output = GLOBALS.set(&Default::default(), || {
+    GLOBALS.set(&Default::default(), || {
         let cm = Arc::<SourceMap>::default();
 
         let globals = &Box::default();
@@ -62,8 +62,7 @@ pub fn bundle_script(script: &str) -> String {
             write(minify, cm.clone(), &module.module, &mut buf);
         }
         buf
-    });
-    String::from_utf8(output).expect("invalid utf8 bundle")
+    })
 }
 
 fn write<W: Write>(minify: bool, cm: Arc<SourceMap>, module: &Module, out: W) {

build/bundlers/src/style.rs

@@ -1,4 +1,4 @@
-use crate::guess_mime;
+use crate::guess_embed;
 use base64::engine::general_purpose::STANDARD;
 use base64::Engine;
 use lightningcss::bundler::{Bundler, FileProvider};
@@ -11,7 +11,7 @@ use std::convert::Infallible;
 use std::fs::read;
 use std::path::Path;
 
-pub fn bundle_style(style: &str) -> String {
+pub fn bundle_style(style: &str) -> Vec<u8> {
     // todo build time?
     let fs = FileProvider::new();
     let mut bundler = Bundler::new(
@@ -49,6 +49,7 @@ pub fn bundle_style(style: &str) -> String {
         })
         .expect("failed to output css")
         .code
+        .into_bytes()
 }
 
 struct InlineUrlVisitor;
@@ -64,7 +65,7 @@ impl<'i> Visitor<'i> for InlineUrlVisitor {
                 eprintln!("Failed to write inline file {path}: {e}");
                 panic!("Failed to inline");
             });
-            let (mime, encode) = guess_mime(path);
+            let (mime, encode) = guess_embed(path);
 
             if encode {
                 let encoded = STANDARD.encode(content);

build/derive/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "demostf-build-derive"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "demostf_build_derive"
+proc-macro = true
+
+[dependencies]
+syn = "2.0.12"
+quote = "1.0.26"
+proc-macro2 = "1.0.54"
+structmeta = "0.2.0"
+merge = "0.1.0"
+demostf-build-bundlers = {version = "0.1", path = "../bundlers"}

build/derive/src/asset.rs

@@ -0,0 +1,203 @@
+use crate::{err, Derivable, DeriveParams};
+use demostf_build_bundlers::guess_mime;
+use merge::Merge;
+use proc_macro2::{Ident, TokenStream};
+use quote::{quote, quote_spanned};
+use structmeta::StructMeta;
+use syn::parse::Parse;
+use syn::{Attribute, DeriveInput, LitStr, Result};
+
+pub struct Asset;
+
+impl Derivable for Asset {
+    type Params = AssetParams;
+
+    fn derive(params: AssetParams) -> Result<TokenStream> {
+        let struct_ident = params.name;
+        let span = struct_ident.span();
+        let input_path = params.source;
+
+        let asset = if params.debug {
+            AssetContent::Runtime {
+                url: input_path.clone(),
+                ty: params.ty,
+            }
+        } else {
+            let content = params.ty.bundle(&input_path);
+            AssetContent::Compiled {
+                route: params.url.clone(),
+                content,
+            }
+        };
+        let content = asset.content();
+        let buster = asset.buster();
+        let etag = asset.etag();
+        let url = asset.url();
+        let mime = guess_mime(&input_path);
+        let mime = quote_spanned!(span => #mime);
+        let route = params.url;
+        let route = quote_spanned!(span => #route);
+
+        Ok(
+            quote_spanned!(span => impl demostf_build::Asset for #struct_ident {
+                fn mime() -> &'static str {
+                    #mime
+                }
+                fn etag() -> &'static str {
+                    #etag
+                }
+                fn cache_buster() -> std::borrow::Cow<'static, str> {
+                    #buster
+                }
+                fn content() -> std::borrow::Cow<'static, [u8]> {
+                    #content
+                }
+                fn route() -> &'static str {
+                    #route
+                }
+                fn url() -> std::borrow::Cow<'static, str> {
+                    #url
+                }
+            }),
+        )
+    }
+}
+
+pub struct AssetParams {
+    name: Ident,
+    debug: bool,
+    source: String,
+    url: String,
+    ty: AssetType,
+}
+
+enum AssetType {
+    Js,
+    Css,
+    Raw,
+}
+
+impl AssetType {
+    fn from_url(url: &str) -> Self {
+        if url.ends_with("css") {
+            return AssetType::Css;
+        } else if url.ends_with("js") {
+            return AssetType::Js;
+        }
+        return AssetType::Raw;
+    }
+
+    fn bundle_fn(&self) -> TokenStream {
+        match self {
+            AssetType::Css => quote!(bundle_style),
+            AssetType::Js => quote!(bundle_script),
+            AssetType::Raw => quote!(bundle_raw),
+        }
+    }
+
+    fn bundle(self, url: &str) -> Vec<u8> {
+        match self {
+            AssetType::Css => demostf_build_bundlers::bundle_style(url),
+            AssetType::Js => demostf_build_bundlers::bundle_script(url),
+            AssetType::Raw => demostf_build_bundlers::bundle_raw(url),
+        }
+    }
+}
+
+enum AssetContent {
+    Runtime { url: String, ty: AssetType },
+    Compiled { route: String, content: Vec<u8> },
+}
+
+impl AssetContent {
+    pub fn content(&self) -> TokenStream {
+        match self {
+            AssetContent::Runtime { url, ty } => {
+                let bundle_fn = ty.bundle_fn();
+                quote!(demostf_build::#bundle_fn(#url).into())
+            }
+            AssetContent::Compiled { content, .. } => {
+                quote!(std::borrow::Cow::Borrowed(&[
+                    #(#content,)*
+                ]))
+            }
+        }
+    }
+    pub fn buster(&self) -> TokenStream {
+        match self {
+            AssetContent::Runtime { .. } => {
+                quote!(demostf_build::random_cache_buster().into())
+            }
+            AssetContent::Compiled { content, .. } => {
+                let hash = demostf_build_bundlers::hash(&content);
+                quote!(std::borrow::Cow::Borrowed(#hash))
+            }
+        }
+    }
+    pub fn etag(&self) -> TokenStream {
+        match self {
+            AssetContent::Runtime { .. } => {
+                quote!("")
+            }
+            AssetContent::Compiled { content, .. } => {
+                let hash = demostf_build_bundlers::hash(&content);
+                quote!(#hash)
+            }
+        }
+    }
+    pub fn url(&self) -> TokenStream {
+        match self {
+            AssetContent::Runtime { .. } => {
+                quote!(format!("{}?v={}", Self::route(), Self::cache_buster()).into())
+            }
+            AssetContent::Compiled { content, route } => {
+                let hash = demostf_build_bundlers::hash(&content);
+                let url = format!("{}?v={}", route, hash);
+                quote!(std::borrow::Cow::Borrowed(#url))
+            }
+        }
+    }
+}
+
+impl DeriveParams for AssetParams {
+    fn parse(input: &DeriveInput) -> Result<AssetParams> {
+        let attrs: AssetAttrs = parse_attrs(&input.attrs);
+        #[cfg(debug_assertions)]
+        let debug = true;
+        #[cfg(not(debug_assertions))]
+        let debug = false;
+        let Some(source) = attrs.source else {
+            return err("missing require attribute #[asset(source)]", input);
+        };
+        let source = source.value();
+        let Some(url) = attrs.url else {
+            return err("missing require attribute #[asset(url)]", input);
+        };
+        let url = url.value();
+        let ty = AssetType::from_url(&url);
+
+        Ok(AssetParams {
+            name: input.ident.clone(),
+            debug,
+            source,
+            url,
+            ty,
+        })
+    }
+}
+
+fn parse_attrs<T: Parse + Default + Merge>(attrs: &[Attribute]) -> T {
+    let mut result = T::default();
+    for attr in attrs {
+        if let Ok(parsed) = attr.parse_args() {
+            result.merge(parsed);
+        }
+    }
+    result
+}
+
+#[derive(Default, StructMeta, Merge)]
+struct AssetAttrs {
+    source: Option<LitStr>,
+    url: Option<LitStr>,
+}

build/derive/src/lib.rs

@@ -0,0 +1,43 @@
+//! Derive macros for tf-log-parser
+
+extern crate proc_macro;
+
+mod asset;
+
+use crate::asset::Asset;
+use proc_macro2::TokenStream;
+use quote::ToTokens;
+use std::fmt::Display;
+use syn::{parse_macro_input, DeriveInput, Error, Result};
+
+/// Derive the `Asset` trait for a struct
+#[proc_macro_derive(Asset, attributes(asset))]
+pub fn derive_asset(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let expanded = derive_trait::<Asset>(parse_macro_input!(input as DeriveInput));
+
+    proc_macro::TokenStream::from(expanded)
+}
+
+/// Basic wrapper for error handling
+fn derive_trait<Trait: Derivable>(input: DeriveInput) -> TokenStream {
+    derive_trait_inner::<Trait>(input).unwrap_or_else(|err| err.into_compile_error())
+}
+
+fn derive_trait_inner<Trait: Derivable>(input: DeriveInput) -> Result<TokenStream> {
+    let params = Trait::Params::parse(&input)?;
+    Trait::derive(params)
+}
+
+trait Derivable {
+    type Params: DeriveParams;
+
+    fn derive(params: Self::Params) -> Result<TokenStream>;
+}
+
+trait DeriveParams: Sized {
+    fn parse(input: &DeriveInput) -> Result<Self>;
+}
+
+fn err<R, T: ToTokens, U: Display>(msg: U, span: T) -> Result<R> {
+    return Err(Error::new_spanned(&span, msg));
+}

build/src/bundle_script.rs

@@ -1,11 +1,9 @@
 use std::env::args;
 
-mod script;
-
 fn main() {
     tracing_subscriber::fmt::init();
 
     let path = args().skip(1).next().unwrap();
-    let output = script::bundle_script(&path);
+    let output = demostf_build::bundle_script(&path);
     // println!("{output}")
 }

build/src/lib.rs

@@ -1,30 +1,23 @@
-mod script;
-mod style;
+pub use demostf_build_bundlers::bundle_raw;
+pub use demostf_build_bundlers::bundle_script;
+pub use demostf_build_bundlers::bundle_style;
+pub use demostf_build_derive::Asset;
+use rand::{distributions::Alphanumeric, Rng};
+use std::borrow::Cow;
 
-use const_fnv1a_hash::fnv1a_hash_str_32;
-pub use script::bundle_script;
-pub use style::bundle_style;
-
-#[macro_export]
-macro_rules! save_asset {
-    ($name:expr, $val:expr) => {
-        let val = $val;
-        let out_dir = std::env::var("OUT_DIR").unwrap();
-        std::fs::write(format!("{out_dir}/{}", $name), &val).expect("failed to write asset");
-        let hash = demostf_build::hash(&val);
-        std::fs::write(format!("{out_dir}/{}.hash", $name), hash)
-            .expect("failed to write asset hash");
-    };
+pub trait Asset {
+    fn mime() -> &'static str;
+    fn cache_buster() -> Cow<'static, str>;
+    fn etag() -> &'static str;
+    fn content() -> Cow<'static, [u8]>;
+    fn url() -> Cow<'static, str>;
+    fn route() -> &'static str;
 }
 
-pub fn hash(data: &str) -> String {
-    format!("{:x}", fnv1a_hash_str_32(data))
-}
-
-fn guess_mime(path: &str) -> (&'static str, bool) {
-    match path.split('.').last().unwrap() {
-        "svg" => ("image/svg+xml", false),
-        "png" => ("image/png", true),
-        ext => panic!("no mimetype known for {ext}"),
-    }
+pub fn random_cache_buster() -> String {
+    rand::thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(8)
+        .map(char::from)
+        .collect()
 }