mirror of
https://codeberg.org/icewind/ugc-scaper.git
synced 2026-06-03 18:24:10 +02:00
team archive
This commit is contained in:
parent
a46314d91a
commit
f4082d619d
15 changed files with 514 additions and 37 deletions
12
archiver/.sqlx/query-0626bad337af6332795252dc9dc2af03eb471cbd38efe61afd712860bcd2fe62.json
generated
Normal file
12
archiver/.sqlx/query-0626bad337af6332795252dc9dc2af03eb471cbd38efe61afd712860bcd2fe62.json
generated
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "INSERT INTO records (\n team_id, season, wins, losses\n ) VALUES ($1, $2, $3, $4)",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Left": ["Int4", "Int4", "Int4", "Int4"]
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "0626bad337af6332795252dc9dc2af03eb471cbd38efe61afd712860bcd2fe62"
|
||||||
|
}
|
||||||
24
archiver/.sqlx/query-4e366cccbfd194d3b52eb7209caaccebb73bf731e3f7a9f274363966925e8ee6.json
generated
Normal file
24
archiver/.sqlx/query-4e366cccbfd194d3b52eb7209caaccebb73bf731e3f7a9f274363966925e8ee6.json
generated
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "INSERT INTO memberships (\n team_id, steam_id, role, since\n ) VALUES ($1, $2, $3, $4)",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Int4",
|
||||||
|
"Int8",
|
||||||
|
{
|
||||||
|
"Custom": {
|
||||||
|
"name": "membership_role",
|
||||||
|
"kind": {
|
||||||
|
"Enum": ["leader", "member"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Timestamptz"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "4e366cccbfd194d3b52eb7209caaccebb73bf731e3f7a9f274363966925e8ee6"
|
||||||
|
}
|
||||||
18
archiver/.sqlx/query-51956c0afc079c48272920a4bd615c47c92908889c4291337bba3cbdd162f2c3.json
generated
Normal file
18
archiver/.sqlx/query-51956c0afc079c48272920a4bd615c47c92908889c4291337bba3cbdd162f2c3.json
generated
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "select id from teams where region IS NULL and format != 'eights' order by id desc",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "id",
|
||||||
|
"type_info": "Int4"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": []
|
||||||
|
},
|
||||||
|
"nullable": [false]
|
||||||
|
},
|
||||||
|
"hash": "51956c0afc079c48272920a4bd615c47c92908889c4291337bba3cbdd162f2c3"
|
||||||
|
}
|
||||||
12
archiver/.sqlx/query-5daea92ea98daa8e37969e9e7cdb7d1e97fea0c6e07f2ba5a2a513fc1bfc41c9.json
generated
Normal file
12
archiver/.sqlx/query-5daea92ea98daa8e37969e9e7cdb7d1e97fea0c6e07f2ba5a2a513fc1bfc41c9.json
generated
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "INSERT INTO team_name_changes (\n team_id, from_tag, from_name, to_tag, to_name, date\n ) VALUES ($1, $2, $3, $4, $5, $6)",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Left": ["Int4", "Varchar", "Varchar", "Varchar", "Varchar", "Date"]
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "5daea92ea98daa8e37969e9e7cdb7d1e97fea0c6e07f2ba5a2a513fc1bfc41c9"
|
||||||
|
}
|
||||||
28
archiver/.sqlx/query-902d2815a624acb69228dd6e7afc698dac51107067d44659592a99493ad5473a.json
generated
Normal file
28
archiver/.sqlx/query-902d2815a624acb69228dd6e7afc698dac51107067d44659592a99493ad5473a.json
generated
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "UPDATE teams SET region = $2 WHERE id = $1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Int4",
|
||||||
|
{
|
||||||
|
"Custom": {
|
||||||
|
"name": "region",
|
||||||
|
"kind": {
|
||||||
|
"Enum": [
|
||||||
|
"europe",
|
||||||
|
"north-america",
|
||||||
|
"south-america",
|
||||||
|
"asia",
|
||||||
|
"australia"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "902d2815a624acb69228dd6e7afc698dac51107067d44659592a99493ad5473a"
|
||||||
|
}
|
||||||
23
archiver/.sqlx/query-a559964ee9c6927333b32de71f1a666fae60ea06cb81a85b71345fb732343fdc.json
generated
Normal file
23
archiver/.sqlx/query-a559964ee9c6927333b32de71f1a666fae60ea06cb81a85b71345fb732343fdc.json
generated
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "select greatest(max(team_home), max(team_away)) as max, least(min(team_home), min(team_away)) as min from matches limit 1;",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "max",
|
||||||
|
"type_info": "Int4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ordinal": 1,
|
||||||
|
"name": "min",
|
||||||
|
"type_info": "Int4"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": []
|
||||||
|
},
|
||||||
|
"nullable": [null, null]
|
||||||
|
},
|
||||||
|
"hash": "a559964ee9c6927333b32de71f1a666fae60ea06cb81a85b71345fb732343fdc"
|
||||||
|
}
|
||||||
12
archiver/.sqlx/query-b153094d4f619005eecae2c23568e10c2f76613e30935b654f67f1a7b07813ea.json
generated
Normal file
12
archiver/.sqlx/query-b153094d4f619005eecae2c23568e10c2f76613e30935b654f67f1a7b07813ea.json
generated
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "INSERT INTO titles (\n team_id, title\n ) VALUES ($1, $2)",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Left": ["Int4", "Varchar"]
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "b153094d4f619005eecae2c23568e10c2f76613e30935b654f67f1a7b07813ea"
|
||||||
|
}
|
||||||
18
archiver/.sqlx/query-dd9ca1267a43207ad70d6cf683291866c889d50dd3793a89936f0731151f8aec.json
generated
Normal file
18
archiver/.sqlx/query-dd9ca1267a43207ad70d6cf683291866c889d50dd3793a89936f0731151f8aec.json
generated
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "SELECT id FROM teams ORDER BY id DESC LIMIT 1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"ordinal": 0,
|
||||||
|
"name": "id",
|
||||||
|
"type_info": "Int4"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Left": []
|
||||||
|
},
|
||||||
|
"nullable": [false]
|
||||||
|
},
|
||||||
|
"hash": "dd9ca1267a43207ad70d6cf683291866c889d50dd3793a89936f0731151f8aec"
|
||||||
|
}
|
||||||
43
archiver/.sqlx/query-e2a0bc541b90e1374bdbaf9f151af569ab906620c2b9da88d919a072c8073f6f.json
generated
Normal file
43
archiver/.sqlx/query-e2a0bc541b90e1374bdbaf9f151af569ab906620c2b9da88d919a072c8073f6f.json
generated
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
{
|
||||||
|
"db_name": "PostgreSQL",
|
||||||
|
"query": "INSERT INTO teams (\n id, tag, name, image, format, region, timezone, steam_group, division, description\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Left": [
|
||||||
|
"Int4",
|
||||||
|
"Varchar",
|
||||||
|
"Varchar",
|
||||||
|
"Varchar",
|
||||||
|
{
|
||||||
|
"Custom": {
|
||||||
|
"name": "game_mode",
|
||||||
|
"kind": {
|
||||||
|
"Enum": ["highlander", "eights", "sixes", "fours", "ultiduo"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Custom": {
|
||||||
|
"name": "region",
|
||||||
|
"kind": {
|
||||||
|
"Enum": [
|
||||||
|
"europe",
|
||||||
|
"north-america",
|
||||||
|
"south-america",
|
||||||
|
"asia",
|
||||||
|
"australia"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Varchar",
|
||||||
|
"Varchar",
|
||||||
|
"Varchar",
|
||||||
|
"Varchar"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "e2a0bc541b90e1374bdbaf9f151af569ab906620c2b9da88d919a072c8073f6f"
|
||||||
|
}
|
||||||
70
archiver/Cargo.lock
generated
70
archiver/Cargo.lock
generated
|
|
@ -87,6 +87,7 @@ name = "archiver"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap",
|
"clap",
|
||||||
|
"futures-util",
|
||||||
"main_error",
|
"main_error",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"secretfile",
|
"secretfile",
|
||||||
|
|
@ -94,6 +95,7 @@ dependencies = [
|
||||||
"sqlx",
|
"sqlx",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
"toml",
|
"toml",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
|
@ -186,9 +188,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.18"
|
version = "1.2.19"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c"
|
checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
@ -201,9 +203,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.5.35"
|
version = "4.5.36"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944"
|
checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap_builder",
|
"clap_builder",
|
||||||
"clap_derive",
|
"clap_derive",
|
||||||
|
|
@ -211,9 +213,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap_builder"
|
name = "clap_builder"
|
||||||
version = "4.5.35"
|
version = "4.5.36"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9"
|
checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anstream",
|
"anstream",
|
||||||
"anstyle",
|
"anstyle",
|
||||||
|
|
@ -537,6 +539,17 @@ version = "0.3.31"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
|
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-macro"
|
||||||
|
version = "0.3.31"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-sink"
|
name = "futures-sink"
|
||||||
version = "0.3.31"
|
version = "0.3.31"
|
||||||
|
|
@ -557,6 +570,7 @@ checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
|
"futures-macro",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"futures-task",
|
"futures-task",
|
||||||
"memchr",
|
"memchr",
|
||||||
|
|
@ -606,9 +620,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.4.8"
|
version = "0.4.9"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
|
checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atomic-waker",
|
"atomic-waker",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
|
@ -1813,9 +1827,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx"
|
name = "sqlx"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4410e73b3c0d8442c5f99b425d7a435b5ee0ae4167b3196771dd3f7a01be745f"
|
checksum = "14e22987355fbf8cfb813a0cf8cd97b1b4ec834b94dbd759a9e8679d41fabe83"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
"sqlx-macros",
|
"sqlx-macros",
|
||||||
|
|
@ -1826,10 +1840,11 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx-core"
|
name = "sqlx-core"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6a007b6936676aa9ab40207cde35daab0a04b823be8ae004368c0793b96a61e0"
|
checksum = "55c4720d7d4cd3d5b00f61d03751c685ad09c33ae8290c8a2c11335e0604300b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64",
|
||||||
"bytes",
|
"bytes",
|
||||||
"crc",
|
"crc",
|
||||||
"crossbeam-queue",
|
"crossbeam-queue",
|
||||||
|
|
@ -1851,6 +1866,7 @@ dependencies = [
|
||||||
"sha2",
|
"sha2",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
|
"time",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
@ -1859,9 +1875,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx-macros"
|
name = "sqlx-macros"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3112e2ad78643fef903618d78cf0aec1cb3134b019730edb039b69eaf531f310"
|
checksum = "175147fcb75f353ac7675509bc58abb2cb291caf0fd24a3623b8f7e3eb0a754b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
|
|
@ -1872,9 +1888,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx-macros-core"
|
name = "sqlx-macros-core"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4e9f90acc5ab146a99bf5061a7eb4976b573f560bc898ef3bf8435448dd5e7ad"
|
checksum = "1cde983058e53bfa75998e1982086c5efe3c370f3250bf0357e344fa3352e32b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"either",
|
"either",
|
||||||
|
|
@ -1898,9 +1914,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx-mysql"
|
name = "sqlx-mysql"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233"
|
checksum = "847d2e5393a4f39e47e4f36cab419709bc2b83cbe4223c60e86e1471655be333"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atoi",
|
"atoi",
|
||||||
"base64",
|
"base64",
|
||||||
|
|
@ -1934,15 +1950,16 @@ dependencies = [
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
|
"time",
|
||||||
"tracing",
|
"tracing",
|
||||||
"whoami",
|
"whoami",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx-postgres"
|
name = "sqlx-postgres"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613"
|
checksum = "cc35947a541b9e0a2e3d85da444f1c4137c13040267141b208395a0d0ca4659f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atoi",
|
"atoi",
|
||||||
"base64",
|
"base64",
|
||||||
|
|
@ -1971,15 +1988,16 @@ dependencies = [
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
"stringprep",
|
"stringprep",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
|
"time",
|
||||||
"tracing",
|
"tracing",
|
||||||
"whoami",
|
"whoami",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sqlx-sqlite"
|
name = "sqlx-sqlite"
|
||||||
version = "0.8.3"
|
version = "0.8.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f85ca71d3a5b24e64e1d08dd8fe36c6c95c339a896cc33068148906784620540"
|
checksum = "6c48291dac4e5ed32da0927a0b981788be65674aeb62666d19873ab4289febde"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"atoi",
|
"atoi",
|
||||||
"flume",
|
"flume",
|
||||||
|
|
@ -1994,6 +2012,8 @@ dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
"serde_urlencoded",
|
"serde_urlencoded",
|
||||||
"sqlx-core",
|
"sqlx-core",
|
||||||
|
"thiserror 2.0.12",
|
||||||
|
"time",
|
||||||
"tracing",
|
"tracing",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
@ -2417,12 +2437,12 @@ checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ugc-scraper-types"
|
name = "ugc-scraper-types"
|
||||||
version = "0.1.2"
|
version = "0.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "55c915ab634173eeefde4bcc7a6af185df442b189665585394784bd9fec64d70"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
|
"sqlx",
|
||||||
"steamid-ng",
|
"steamid-ng",
|
||||||
|
"thiserror 2.0.12",
|
||||||
"time",
|
"time",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,8 @@ version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
ugc-scraper-types = "0.1.2"
|
# ugc-scraper-types = "0.1.2"
|
||||||
|
ugc-scraper-types = { version = "0.2.0", path = "../types", features = ["sqlx"] }
|
||||||
reqwest = { version = "0.12.15", features = ["json"] }
|
reqwest = { version = "0.12.15", features = ["json"] }
|
||||||
clap = { version = "4.5.35", features = ["derive"] }
|
clap = { version = "4.5.35", features = ["derive"] }
|
||||||
tracing = "0.1.41"
|
tracing = "0.1.41"
|
||||||
|
|
@ -13,6 +14,8 @@ serde = { version = "1.0.219", features = ["derive"] }
|
||||||
toml = "0.8.20"
|
toml = "0.8.20"
|
||||||
secretfile = "0.1.0"
|
secretfile = "0.1.0"
|
||||||
tokio = { version = "1.44.2", features = ["macros", "rt-multi-thread"] }
|
tokio = { version = "1.44.2", features = ["macros", "rt-multi-thread"] }
|
||||||
sqlx = { version = "0.8.3", features = ["postgres", "runtime-tokio"] }
|
sqlx = { version = "0.8.3", features = ["postgres", "runtime-tokio", "time"] }
|
||||||
thiserror = "2.0.12"
|
thiserror = "2.0.12"
|
||||||
main_error = "0.1.2"
|
main_error = "0.1.2"
|
||||||
|
tokio-stream = "0.1.17"
|
||||||
|
futures-util = "0.3.31"
|
||||||
|
|
@ -5,11 +5,12 @@ CREATE TYPE membership_role AS ENUM ('leader', 'member');
|
||||||
CREATE TABLE teams
|
CREATE TABLE teams
|
||||||
(
|
(
|
||||||
id INTEGER NOT NULL,
|
id INTEGER NOT NULL,
|
||||||
|
tag VARCHAR NOT NULL,
|
||||||
name VARCHAR NOT NULL,
|
name VARCHAR NOT NULL,
|
||||||
image VARCHAR NOT NULL,
|
image VARCHAR,
|
||||||
format game_mode NOT NULL,
|
format game_mode NOT NULL,
|
||||||
region region,
|
region region,
|
||||||
timezone VARCHAR NOT NULL,
|
timezone VARCHAR,
|
||||||
steam_group VARCHAR,
|
steam_group VARCHAR,
|
||||||
division VARCHAR NOT NULL,
|
division VARCHAR NOT NULL,
|
||||||
description VARCHAR NOT NULL
|
description VARCHAR NOT NULL
|
||||||
|
|
@ -39,7 +40,7 @@ CREATE TABLE titles
|
||||||
CREATE INDEX titles_team_id_idx
|
CREATE INDEX titles_team_id_idx
|
||||||
ON titles USING BTREE (team_id);
|
ON titles USING BTREE (team_id);
|
||||||
|
|
||||||
CREATE TABLE name_changes
|
CREATE TABLE team_name_changes
|
||||||
(
|
(
|
||||||
team_id INTEGER NOT NULL,
|
team_id INTEGER NOT NULL,
|
||||||
from_tag VARCHAR NOT NULL,
|
from_tag VARCHAR NOT NULL,
|
||||||
|
|
@ -49,8 +50,8 @@ CREATE TABLE name_changes
|
||||||
date DATE
|
date DATE
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX name_changes_team_id_idx
|
CREATE INDEX team_name_changes_team_id_idx
|
||||||
ON name_changes USING BTREE (team_id);
|
ON team_name_changes USING BTREE (team_id);
|
||||||
|
|
||||||
CREATE TABLE records
|
CREATE TABLE records
|
||||||
(
|
(
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,13 @@
|
||||||
|
use futures_util::stream::TryStreamExt;
|
||||||
use sqlx::postgres::PgConnectOptions;
|
use sqlx::postgres::PgConnectOptions;
|
||||||
use sqlx::{query, Error, PgPool};
|
use sqlx::{query, Error, Executor, PgPool, Postgres};
|
||||||
|
use std::ops::Range;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use ugc_scraper_types::MatchInfo;
|
use tokio_stream::Stream;
|
||||||
|
use ugc_scraper_types::{
|
||||||
|
GameMode, MatchInfo, Membership, MembershipRole, NameChange, Record, Region, Team,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum ArchiveError {
|
pub enum ArchiveError {
|
||||||
|
|
@ -70,4 +75,197 @@ impl Archive {
|
||||||
})?
|
})?
|
||||||
.map(|row| row.id as u32))
|
.map(|row| row.id as u32))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn store_team(&self, id: u32, team: &Team) -> Result<(), ArchiveError> {
|
||||||
|
let mut transaction = self
|
||||||
|
.pool
|
||||||
|
.begin()
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "beginning team transaction",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
query!(
|
||||||
|
"INSERT INTO teams (
|
||||||
|
id, tag, name, image, format, region, timezone, steam_group, division, description
|
||||||
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
|
||||||
|
id as i32,
|
||||||
|
team.tag,
|
||||||
|
team.name,
|
||||||
|
team.image,
|
||||||
|
team.format as GameMode,
|
||||||
|
team.region as Option<Region>,
|
||||||
|
team.timezone,
|
||||||
|
team.steam_group,
|
||||||
|
team.division,
|
||||||
|
team.description,
|
||||||
|
)
|
||||||
|
.execute(&mut *transaction)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "inserting team",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
|
||||||
|
for title in team.titles.iter() {
|
||||||
|
Self::store_title(&mut *transaction, id, title).await?;
|
||||||
|
}
|
||||||
|
for name_change in team.name_changes.iter() {
|
||||||
|
Self::store_team_name_change(&mut *transaction, id, name_change).await?
|
||||||
|
}
|
||||||
|
for record in team.results.iter() {
|
||||||
|
Self::store_record(&mut *transaction, id, record).await?
|
||||||
|
}
|
||||||
|
for membership in team.members.iter() {
|
||||||
|
Self::store_membership(&mut *transaction, id, membership).await?
|
||||||
|
}
|
||||||
|
|
||||||
|
transaction
|
||||||
|
.commit()
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "commiting team transaction",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn update_team_region(&self, id: u32, team: &Team) -> Result<(), ArchiveError> {
|
||||||
|
query!(
|
||||||
|
"UPDATE teams SET region = $2 WHERE id = $1",
|
||||||
|
id as i32,
|
||||||
|
team.region as Option<Region>,
|
||||||
|
)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "updating team region",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_last_team_id(&self) -> Result<Option<u32>, ArchiveError> {
|
||||||
|
Ok(query!("SELECT id FROM teams ORDER BY id DESC LIMIT 1")
|
||||||
|
.fetch_optional(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "getting latest team",
|
||||||
|
error,
|
||||||
|
})?
|
||||||
|
.map(|row| row.id as u32))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_team_range(&self) -> Result<Range<u32>, ArchiveError> {
|
||||||
|
let row = query!("select greatest(max(team_home), max(team_away)) as max, least(min(team_home), min(team_away)) as min from matches limit 1;")
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "getting latest team",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok((row.min.unwrap_or_default() as u32)..(row.max.unwrap_or_default() as u32))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_no_region_teams(&self) -> impl Stream<Item = Result<u32, ArchiveError>> + use<'_> {
|
||||||
|
query!("select id from teams where region IS NULL and format != 'eights' order by id desc")
|
||||||
|
.fetch(&self.pool)
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "getting teams without region",
|
||||||
|
error,
|
||||||
|
})
|
||||||
|
.map_ok(|map| map.id as u32)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_title(
|
||||||
|
db: impl Executor<'_, Database = Postgres>,
|
||||||
|
team_id: u32,
|
||||||
|
title: &str,
|
||||||
|
) -> Result<(), ArchiveError> {
|
||||||
|
query!(
|
||||||
|
"INSERT INTO titles (
|
||||||
|
team_id, title
|
||||||
|
) VALUES ($1, $2)",
|
||||||
|
team_id as i32,
|
||||||
|
title
|
||||||
|
)
|
||||||
|
.execute(db)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "inserting title",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_team_name_change(
|
||||||
|
db: impl Executor<'_, Database = Postgres>,
|
||||||
|
team_id: u32,
|
||||||
|
change: &NameChange,
|
||||||
|
) -> Result<(), ArchiveError> {
|
||||||
|
query!(
|
||||||
|
"INSERT INTO team_name_changes (
|
||||||
|
team_id, from_tag, from_name, to_tag, to_name, date
|
||||||
|
) VALUES ($1, $2, $3, $4, $5, $6)",
|
||||||
|
team_id as i32,
|
||||||
|
change.from_tag,
|
||||||
|
change.from,
|
||||||
|
change.to_tag,
|
||||||
|
change.to,
|
||||||
|
change.date
|
||||||
|
)
|
||||||
|
.execute(db)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "inserting name change",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_membership(
|
||||||
|
db: impl Executor<'_, Database = Postgres>,
|
||||||
|
team_id: u32,
|
||||||
|
membership: &Membership,
|
||||||
|
) -> Result<(), ArchiveError> {
|
||||||
|
query!(
|
||||||
|
"INSERT INTO memberships (
|
||||||
|
team_id, steam_id, role, since
|
||||||
|
) VALUES ($1, $2, $3, $4)",
|
||||||
|
team_id as i32,
|
||||||
|
u64::from(membership.steam_id) as i64,
|
||||||
|
membership.role as MembershipRole,
|
||||||
|
membership.since,
|
||||||
|
)
|
||||||
|
.execute(db)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "inserting membership",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn store_record(
|
||||||
|
db: impl Executor<'_, Database = Postgres>,
|
||||||
|
team_id: u32,
|
||||||
|
record: &Record,
|
||||||
|
) -> Result<(), ArchiveError> {
|
||||||
|
query!(
|
||||||
|
"INSERT INTO records (
|
||||||
|
team_id, season, wins, losses
|
||||||
|
) VALUES ($1, $2, $3, $4)",
|
||||||
|
team_id as i32,
|
||||||
|
record.season as i32,
|
||||||
|
record.wins as i32,
|
||||||
|
record.losses as i32,
|
||||||
|
)
|
||||||
|
.execute(db)
|
||||||
|
.await
|
||||||
|
.map_err(|error| ArchiveError::Query {
|
||||||
|
description: "inserting record",
|
||||||
|
error,
|
||||||
|
})?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,10 @@ use crate::config::Config;
|
||||||
use clap::{Parser, Subcommand};
|
use clap::{Parser, Subcommand};
|
||||||
use main_error::MainResult;
|
use main_error::MainResult;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::pin::pin;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::time::sleep;
|
use tokio::time::sleep;
|
||||||
|
use tokio_stream::StreamExt;
|
||||||
use tracing::{error, info, span, warn, Level};
|
use tracing::{error, info, span, warn, Level};
|
||||||
|
|
||||||
#[derive(Debug, Parser)]
|
#[derive(Debug, Parser)]
|
||||||
|
|
@ -23,6 +25,8 @@ struct Args {
|
||||||
#[derive(Debug, Subcommand)]
|
#[derive(Debug, Subcommand)]
|
||||||
enum Command {
|
enum Command {
|
||||||
Matches,
|
Matches,
|
||||||
|
Teams,
|
||||||
|
FixupTeams,
|
||||||
}
|
}
|
||||||
|
|
||||||
const LAST_MATCH: u32 = 117047;
|
const LAST_MATCH: u32 = 117047;
|
||||||
|
|
@ -40,6 +44,12 @@ async fn main() -> MainResult {
|
||||||
Command::Matches => {
|
Command::Matches => {
|
||||||
archive_matches(&client, &archive).await?;
|
archive_matches(&client, &archive).await?;
|
||||||
}
|
}
|
||||||
|
Command::Teams => {
|
||||||
|
archive_teams(&client, &archive).await?;
|
||||||
|
}
|
||||||
|
Command::FixupTeams => {
|
||||||
|
fixup_teams(&client, &archive).await?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -69,6 +79,61 @@ async fn archive_matches(client: &UgcClient, archive: &Archive) -> MainResult {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn archive_teams(client: &UgcClient, archive: &Archive) -> MainResult {
|
||||||
|
let range = archive.get_team_range().await?;
|
||||||
|
let next_team = archive.get_last_team_id().await?.unwrap_or(range.start - 1) + 1;
|
||||||
|
|
||||||
|
for id in next_team..=range.end {
|
||||||
|
let _span = span!(Level::INFO, "archive_team", id = id).entered();
|
||||||
|
match client.get_team(id).await.check_not_found() {
|
||||||
|
Ok(Some(team_data)) => {
|
||||||
|
if team_data.format.is_tf2() {
|
||||||
|
info!("storing team");
|
||||||
|
archive.store_team(id, &team_data).await?;
|
||||||
|
} else {
|
||||||
|
info!("skipping non-tf2 team");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
warn!("team not found");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("error fetching team: {:?}", e);
|
||||||
|
panic!();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sleep(Duration::from_millis(500)).await;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fixup_teams(client: &UgcClient, archive: &Archive) -> MainResult {
|
||||||
|
let mut ids = pin!(archive.get_no_region_teams());
|
||||||
|
|
||||||
|
while let Some(Ok(id)) = ids.next().await {
|
||||||
|
let _span = span!(Level::INFO, "fixup_team", id = id).entered();
|
||||||
|
match client.get_team(id).await.check_not_found() {
|
||||||
|
Ok(Some(team_data)) => {
|
||||||
|
if team_data.format.is_tf2() {
|
||||||
|
info!(region = ?team_data.region, "updating team region");
|
||||||
|
archive.update_team_region(id, &team_data).await?;
|
||||||
|
} else {
|
||||||
|
info!("skipping non-tf2 team");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
warn!("team not found");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("error fetching team: {:?}", e);
|
||||||
|
panic!();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sleep(Duration::from_millis(500)).await;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
trait NotFoundResultExt<T>: Sized {
|
trait NotFoundResultExt<T>: Sized {
|
||||||
fn check_not_found(self) -> Result<Option<T>, UgcClientError>;
|
fn check_not_found(self) -> Result<Option<T>, UgcClientError>;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -151,8 +151,8 @@ impl Parser for TeamParser {
|
||||||
.map(String::from)
|
.map(String::from)
|
||||||
});
|
});
|
||||||
|
|
||||||
let format_text = select_text(root, &self.selector_team_format)
|
let format_text =
|
||||||
.ok_or(ParseError::ElementNotFound {
|
select_text(root, &self.selector_team_format).ok_or(ParseError::ElementNotFound {
|
||||||
selector: SELECTOR_TEAM_FORMAT,
|
selector: SELECTOR_TEAM_FORMAT,
|
||||||
role: "team format",
|
role: "team format",
|
||||||
})?;
|
})?;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue