use new material loading

This commit is contained in:
Robin Appelman 2023-12-19 18:36:00 +01:00
commit 874266cc38
9 changed files with 250 additions and 195 deletions

91
Cargo.lock generated
View file

@ -196,6 +196,12 @@ dependencies = [
"backtrace",
]
[[package]]
name = "beef"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
[[package]]
name = "binrw"
version = "0.13.3"
@ -1174,6 +1180,38 @@ version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "logos"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1"
dependencies = [
"logos-derive",
]
[[package]]
name = "logos-codegen"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68"
dependencies = [
"beef",
"fnv",
"proc-macro2",
"quote",
"regex-syntax 0.6.29",
"syn 2.0.41",
]
[[package]]
name = "logos-derive"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e"
dependencies = [
"logos-codegen",
]
[[package]]
name = "lzma"
version = "0.2.2"
@ -1295,12 +1333,6 @@ dependencies = [
"syn 2.0.41",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.3.7"
@ -1402,16 +1434,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
@ -2188,14 +2210,6 @@ dependencies = [
"winreg",
]
[[package]]
name = "steamy-vdf"
version = "0.3.0"
source = "git+https://github.com/icewind1991/steamy?branch=nom7#7f9b91c42f857f13bd45a16c7bab1972dafe26d1"
dependencies = [
"nom",
]
[[package]]
name = "strict-num"
version = "0.1.1"
@ -2388,8 +2402,6 @@ dependencies = [
[[package]]
name = "three-d"
version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2db9010227411ab0aa5948e770304e807e5c9b6d5d0719c3de248bae7be7096"
dependencies = [
"cgmath",
"egui",
@ -2661,12 +2673,12 @@ dependencies = [
"cgmath",
"clap",
"delaunator",
"image",
"itertools 0.10.5",
"miette",
"splines",
"steamid-ng",
"steamlocate",
"steamy-vdf",
"tf-demo-parser",
"thiserror",
"three-d",
@ -2676,10 +2688,23 @@ dependencies = [
"tracing-tree",
"vbsp",
"vmdl",
"vmt-parser",
"vpk",
"vtf",
]
[[package]]
name = "vdf-reader"
version = "0.1.0"
source = "git+https://github.com/icewind1991/vdf-reader#d14f22d1bc97cd57c35585b9545c1491f32189bf"
dependencies = [
"logos",
"miette",
"parse-display",
"serde",
"thiserror",
]
[[package]]
name = "vec_map"
version = "0.8.2"
@ -2707,6 +2732,18 @@ dependencies = [
"tracing",
]
[[package]]
name = "vmt-parser"
version = "0.1.0"
source = "git+https://github.com/icewind1991/vmt-parser#8f31c601e56624482b541849f099ff233bb3e2b8"
dependencies = [
"miette",
"serde",
"serde_repr",
"thiserror",
"vdf-reader",
]
[[package]]
name = "vpk"
version = "0.2.0"

View file

@ -6,7 +6,8 @@ authors = ["Robin Appelman <robin@icewind.nl>"]
license = "MIT"
[dependencies]
three-d = { version = "0.16.3", features = ["egui-gui"] }
#three-d = { version = "0.16.3", features = ["egui-gui"] }
three-d = { version = "0.16.3", features = ["egui-gui"], path = "../rust/three-d" }
three-d-asset = { version = "0.6" }
vbsp = { version = "0.2.0", git = "https://github.com/icewind1991/vbsp" }
#vbsp = { version = "0.2.0", path = "../bsp" }
@ -27,7 +28,8 @@ steamid-ng = "1.0.0"
clap = { version = "4.0.29", features = ["derive"] }
splines = { version = "4.1.1", features = ["cgmath"] }
vtf = "0.1.6"
steamy-vdf = { version = "0.3.0", git = "https://github.com/icewind1991/steamy", branch = "nom7" }
vmt-parser = { version = "0.1", git = "https://github.com/icewind1991/vmt-parser" }
image = "0.23.14"
[profile.dev.package."*"]
opt-level = 2

View file

@ -101,6 +101,7 @@
clippy
cargo-audit
cargo-msrv
cargo-flamegraph
]) ++ (buildDependencies pkgs) ++ (runtimeDependencies pkgs);
LD_LIBRARY_PATH = with pkgs; "/run/opengl-driver/lib/:${lib.makeLibraryPath ([libGL libGLU])}";

View file

@ -1,4 +1,4 @@
use crate::material::load_material_fallback;
use crate::material::{convert_material, load_material_fallback};
use crate::prop::load_props;
use crate::{Error, Loader};
use cgmath::Matrix4;
@ -85,6 +85,7 @@ fn model_to_model(model: Handle<vbsp::data::Model>, loader: &Loader) -> CpuModel
let materials: Vec<_> = textures
.iter()
.map(|texture| load_material_fallback(texture, &["".into()], loader))
.map(convert_material)
.collect();
CpuModel {
@ -99,7 +100,10 @@ fn load_world(data: &[u8], loader: &mut Loader) -> Result<(CpuModel, Bsp), Error
loader.set_pack(bsp.pack.clone());
let world_model = bsp.models().next().ok_or(Error::Other("No world model"))?;
let world_model = bsp
.models()
.next()
.ok_or(Error::Other("No world model".into()))?;
Ok((model_to_model(world_model, loader), bsp))
}

View file

@ -1,10 +1,9 @@
use crate::Error;
use std::env::var;
use std::fmt::{Debug, Formatter};
use std::fs;
use std::path::PathBuf;
use steamlocate::SteamDir;
use tracing::{debug, error, info};
use tracing::{debug, info};
use vbsp::Packfile;
use vpk::VPK;
@ -25,33 +24,16 @@ impl Debug for Loader {
impl Loader {
pub fn new() -> Result<Self, Error> {
Self::with_opt_pack(None)
}
#[allow(dead_code)]
pub fn with_pak(pack: Packfile) -> Result<Self, Error> {
Self::with_opt_pack(Some(pack))
}
pub fn with_opt_pack(pack: Option<Packfile>) -> Result<Self, Error> {
let tf2_dir = match var("TF_DIR") {
Ok(dir) => PathBuf::from(dir),
Err(_) => SteamDir::locate()
let tf_dir = SteamDir::locate()
.ok_or("Can't find steam directory")?
.app(&440)
.ok_or("Can't find tf2 directory")?
.path
.clone(),
};
let tf_dir = tf2_dir.join("tf");
let vpk_dirs = [tf_dir.clone(), tf2_dir.join("hl2")];
.join("tf");
let download = tf_dir.join("download");
let vpks = vpk_dirs
.iter()
.flat_map(|dir| dir.read_dir())
.flatten()
.flatten()
let vpks = tf_dir
.read_dir()?
.filter_map(|item| item.ok())
.filter_map(|item| Some(item.path().to_str()?.to_string()))
.filter(|path| path.ends_with("dir.vpk"))
.map(|path| vpk::from_path(&path))
@ -59,10 +41,10 @@ impl Loader {
.collect();
Ok(Loader {
pack,
tf_dir,
download,
vpks,
pack: None,
})
}
@ -70,6 +52,32 @@ impl Loader {
self.pack = Some(pack);
}
#[tracing::instrument]
pub fn exists(&self, name: &str) -> bool {
debug!("loading {}", name);
if name.ends_with("bsp") {
let path = self.tf_dir.join(name);
if path.exists() {
return true;
}
let path = self.download.join(name);
if path.exists() {
return true;
}
}
for vpk in self.vpks.iter() {
if vpk.tree.contains_key(name) {
return true;
}
}
if let Some(pack) = &self.pack {
if let Some(_) = pack.get(name).ok().flatten() {
return true;
}
}
false
}
#[tracing::instrument]
pub fn load(&self, name: &str) -> Result<Vec<u8>, Error> {
debug!("loading {}", name);
@ -102,13 +110,13 @@ impl Loader {
Err(Error::ResourceNotFound(name.to_string()))
}
pub fn load_from_paths(&self, name: &str, paths: &[String]) -> Result<Vec<u8>, Error> {
pub fn find_in_paths(&self, name: &str, paths: &[String]) -> Option<String> {
for path in paths {
if let Ok(data) = self.load(&format!("{}{}", path, name)) {
return Ok(data);
let full_path = format!("{}{}", path, name);
if self.exists(&full_path) {
return Some(full_path);
}
}
error!("Failed to find {} in vpk paths: {}", name, paths.join(", "));
Err(Error::ResourceNotFound(name.to_string()))
None
}
}

View file

@ -23,6 +23,7 @@ use thiserror::Error;
use three_d::*;
use tracing_subscriber::{prelude::*, EnvFilter};
use tracing_tree::HierarchicalLayer;
use vmt_parser::VdfError;
/// View a demo file
#[derive(Parser, Debug)]
@ -47,13 +48,13 @@ pub enum Error {
#[error(transparent)]
Vtf(#[from] vtf::Error),
#[error(transparent)]
Vdf(#[from] steamy_vdf::Error),
Vdf(#[from] VdfError),
#[error(transparent)]
Mdl(#[from] vmdl::ModelError),
#[error(transparent)]
Demo(#[from] tf_demo_parser::ParseError),
#[error("{0}")]
Other(&'static str),
Other(String),
#[error(transparent)]
Window(#[from] WindowError),
#[error(transparent)]
@ -66,7 +67,7 @@ pub enum Error {
impl From<&'static str> for Error {
fn from(e: &'static str) -> Self {
Error::Other(e)
Error::Other(e.into())
}
}

View file

@ -1,137 +1,121 @@
use crate::loader::Loader;
use crate::Error;
use std::str::FromStr;
use steamy_vdf::{Entry, Table};
use three_d::{CpuMaterial, CpuTexture, TextureData};
use image::{DynamicImage, GenericImageView};
use three_d::{CpuMaterial, CpuTexture};
use three_d_asset::Srgba;
use tracing::error;
use tracing::{error, instrument};
use vmt_parser::from_str;
use vmt_parser::material::{Material, WaterMaterial};
use vtf::vtf::VTF;
pub fn load_material_fallback(name: &str, search_dirs: &[String], loader: &Loader) -> CpuMaterial {
pub fn load_material_fallback(name: &str, search_dirs: &[String], loader: &Loader) -> MaterialData {
match load_material(name, search_dirs, loader) {
Ok(material) => material,
Ok(mat) => mat,
Err(e) => {
error!(
material = name,
error = ?e,
"failed to load material, falling back"
);
CpuMaterial {
albedo: Srgba {
r: 255,
g: 0,
b: 255,
a: 255,
},
error!(error = ?e, material = name, "failed to load material");
MaterialData {
name: name.into(),
..Default::default()
color: [255, 0, 255, 255],
..MaterialData::default()
}
}
}
}
fn get_path(vmt: &Entry, name: &str) -> Option<String> {
Some(vmt.lookup(name)?.as_str()?.replace('\\', "/"))
#[derive(Default, Debug)]
pub struct MaterialData {
pub name: String,
pub path: String,
pub color: [u8; 4],
pub texture: Option<TextureData>,
pub alpha_test: Option<f32>,
pub bump_map: Option<TextureData>,
pub translucent: bool,
}
#[derive(Debug)]
pub struct TextureData {
pub name: String,
pub image: DynamicImage,
}
#[instrument(skip(loader))]
pub fn load_material(
name: &str,
search_dirs: &[String],
loader: &Loader,
) -> Result<CpuMaterial, Error> {
) -> Result<MaterialData, Error> {
let dirs = search_dirs
.iter()
.map(|dir| {
format!(
"materials/{}",
dir.to_ascii_lowercase().trim_start_matches('/')
dir.to_ascii_lowercase().trim_start_matches("/")
)
})
.collect::<Vec<_>>();
let path = format!("{}.vmt", name.to_ascii_lowercase().trim_end_matches(".vmt"));
let raw = loader.load_from_paths(&path, &dirs)?.to_ascii_lowercase();
let path = loader
.find_in_paths(&path, &dirs)
.ok_or(Error::ResourceNotFound(path))?;
let raw = loader.load(&path)?;
let vdf = String::from_utf8(raw)?;
let vmt = parse_vdf(&raw)?;
let vmt = resolve_vmt_patch(vmt, loader)?;
let material = from_str(&vdf).map_err(|e| {
let report = miette::ErrReport::new(e);
println!("{:?}", report);
Error::Other(format!("Failed to load material {}", path))
})?;
let material = material.resolve(|path| {
let data = loader.load(path)?;
let vdf = String::from_utf8(data)?;
Ok::<_, Error>(vdf)
})?;
let material_type = vmt
.keys()
.next()
.ok_or(Error::Other("empty vmt"))?
.to_ascii_lowercase();
if material_type == "water" {
return Ok(CpuMaterial {
albedo: Srgba {
r: 82,
g: 180,
b: 217,
a: 128,
},
if let Material::Water(WaterMaterial {
base_texture: None, ..
}) = &material
{
return Ok(MaterialData {
color: [82, 180, 217, 128],
name: name.into(),
..Default::default()
path,
texture: None,
bump_map: None,
alpha_test: None,
translucent: true,
});
}
let table = vmt
.values()
.next()
.cloned()
.ok_or(Error::Other("empty vmt"))?;
let base_texture = get_path(&table, "$basetexture").ok_or(Error::Other("no $basetexture"))?;
let base_texture = material.base_texture();
let translucent = table
.lookup("$translucent")
.map(|val| val.as_str() == Some("1"))
.unwrap_or_default();
let glass = table
.lookup("$surfaceprop")
.map(|val| val.as_str() == Some("glass"))
.unwrap_or_default();
let alpha_test = table
.lookup("$alphatest")
.map(|val| val.as_str() == Some("1"))
.unwrap_or_default();
let texture = load_texture(
base_texture.as_str(),
loader,
translucent | glass | alpha_test,
)?;
let translucent = material.translucent();
let glass = material.surface_prop() == Some("glass");
let alpha_test = material.alpha_test();
let texture = load_texture(base_texture, loader)?;
let alpha_cutout = table
.lookup("$alphatestreference")
.and_then(Entry::as_str)
.and_then(|val| f32::from_str(val).ok())
.unwrap_or(1.0);
let bump_map = material.bump_map().and_then(|path| {
Some(TextureData {
image: load_texture(&path, loader).ok()?,
name: path.into(),
})
});
let bump_map = get_path(&table, "$bumpmap")
.map(|path| load_texture(&path, loader, true).ok())
.flatten();
Ok(CpuMaterial {
Ok(MaterialData {
color: [255; 4],
name: name.into(),
albedo: Srgba::WHITE,
albedo_texture: Some(texture),
alpha_cutout: alpha_test.then_some(alpha_cutout),
normal_texture: bump_map,
..CpuMaterial::default()
path,
texture: Some(TextureData {
name: base_texture.into(),
image: texture,
}),
bump_map,
alpha_test,
translucent: translucent | glass,
})
}
fn parse_vdf(bytes: &[u8]) -> Result<Table, Error> {
#[cfg(feature = "dump_materials")]
println!("{}", String::from_utf8_lossy(bytes));
let mut reader = steamy_vdf::Reader::from(bytes);
Table::load(&mut reader).map_err(|e| {
error!(
source = String::from_utf8_lossy(bytes).to_string(),
"failed to parse vmt"
);
e.into()
})
}
fn load_texture(name: &str, loader: &Loader, alpha: bool) -> Result<CpuTexture, Error> {
fn load_texture(name: &str, loader: &Loader) -> Result<DynamicImage, Error> {
let path = format!(
"materials/{}.vtf",
name.trim_end_matches(".vtf").trim_start_matches('/')
@ -139,40 +123,53 @@ fn load_texture(name: &str, loader: &Loader, alpha: bool) -> Result<CpuTexture,
let mut raw = loader.load(&path)?;
let vtf = VTF::read(&mut raw)?;
let image = vtf.highres_image.decode(0)?;
let texture_data = if alpha {
TextureData::RgbaU8(image.into_rgba8().pixels().map(|pixel| pixel.0).collect())
Ok(image)
}
pub fn convert_material(material: MaterialData) -> CpuMaterial {
CpuMaterial {
albedo: Srgba::new(
material.color[0],
material.color[1],
material.color[2],
material.color[3],
),
name: material.name,
albedo_texture: material
.texture
.map(|tex| convert_texture(tex, material.translucent | material.alpha_test.is_some())),
alpha_cutout: material.alpha_test,
normal_texture: material.bump_map.map(|tex| convert_texture(tex, true)),
..CpuMaterial::default()
}
}
pub fn convert_texture(texture: TextureData, keep_alpha: bool) -> CpuTexture {
let width = texture.image.width();
let height = texture.image.height();
let data = if keep_alpha {
three_d_asset::TextureData::RgbaU8(
texture
.image
.into_rgba8()
.pixels()
.map(|pixel| pixel.0)
.collect(),
)
} else {
TextureData::RgbU8(image.into_rgb8().pixels().map(|pixel| pixel.0).collect())
three_d_asset::TextureData::RgbU8(
texture
.image
.into_rgb8()
.pixels()
.map(|pixel| pixel.0)
.collect(),
)
};
Ok(CpuTexture {
name: name.into(),
data: texture_data,
height: vtf.header.height as u32,
width: vtf.header.width as u32,
CpuTexture {
data,
name: texture.name,
height,
width,
..CpuTexture::default()
})
}
fn resolve_vmt_patch(vmt: Table, loader: &Loader) -> Result<Table, Error> {
if vmt.len() != 1 {
panic!("vmt with more than 1 item?");
}
if let Some(Entry::Table(patch)) = vmt.get("patch") {
let include = patch
.get("include")
.ok_or(Error::Other("no include in patch"))?
.as_str()
.ok_or(Error::Other("include is not a string"))?;
let _replace = patch
.get("replace")
.ok_or(Error::Other("no replace in patch"))?
.as_table()
.ok_or(Error::Other("replace is not a table"))?;
let included_raw = loader.load(include)?.to_ascii_lowercase();
// todo actually patch
parse_vdf(&included_raw)
} else {
Ok(vmt)
}
}

View file

@ -1,5 +1,5 @@
use crate::bsp::map_coords;
use crate::material::load_material_fallback;
use crate::material::{convert_material, load_material_fallback};
use crate::{Error, Loader};
use three_d::{CpuMaterial, CpuModel, Mat4, Positions, Vec2, Vec3, Vec4};
use three_d_asset::{Geometry, Primitive, TriMesh};
@ -115,5 +115,9 @@ fn prop_to_meshes<'a>(prop: &'a PropData) -> impl Iterator<Item = Primitive> + '
}
fn prop_texture_to_material(texture: &TextureInfo, loader: &Loader) -> CpuMaterial {
load_material_fallback(&texture.name, &texture.search_paths, loader)
convert_material(load_material_fallback(
&texture.name,
&texture.search_paths,
loader,
))
}

View file

@ -1,11 +1,12 @@
use crate::control::{Control, DebugToggle};
use crate::ui::DebugType;
use crate::DebugUI;
use std::sync::Arc;
use three_d::*;
pub struct Renderer<C: Control> {
gui: DebugUI,
pub models: Vec<Model<PhysicalMaterial>>,
pub models: Vec<Model<Arc<PhysicalMaterial>>>,
ambient_lights: Vec<AmbientLight>,
directional_lights: Vec<DirectionalLight>,
pub context: Context,