support atom feeds

This commit is contained in:
Robin Appelman 2024-04-05 16:28:55 +02:00
commit 40b1d77bdd
3 changed files with 258 additions and 77 deletions

298
Cargo.lock generated
View file

@ -27,16 +27,41 @@ dependencies = [
]
[[package]]
name = "atom_syndication"
version = "0.12.2"
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "571832dcff775e26562e8e6930cd483de5587301d40d3a3b85d532b6383e15a7"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "atom_syndication"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a9a7ab83635ff7a3b04856f4ad95324dccc9b947ab1e790fc5c769ee6d6f60c"
dependencies = [
"derive_builder 0.5.1",
"failure",
"quick-xml 0.12.4",
]
[[package]]
name = "atom_syndication"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d5016bf52ff4f3ed28bf3ec1fed96b53daf4b137d5e6b9f97a8cfae7b57a3a2"
dependencies = [
"chrono",
"derive_builder",
"derive_builder 0.9.0",
"diligent-date-parser",
"never",
"quick-xml",
"quick-xml 0.20.0",
]
[[package]]
@ -96,7 +121,12 @@ version = "0.4.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"wasm-bindgen",
"windows-targets 0.52.4",
]
[[package]]
@ -127,10 +157,16 @@ dependencies = [
]
[[package]]
name = "darling"
version = "0.14.4"
name = "core-foundation-sys"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
[[package]]
name = "darling"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
dependencies = [
"darling_core",
"darling_macro",
@ -138,26 +174,26 @@ dependencies = [
[[package]]
name = "darling_core"
version = "0.14.4"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"quote 1.0.35",
"strsim",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.14.4"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
dependencies = [
"darling_core",
"quote",
"quote 1.0.35",
"syn 1.0.109",
]
@ -173,32 +209,47 @@ dependencies = [
[[package]]
name = "derive_builder"
version = "0.12.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
checksum = "8c998e6ab02a828dd9735c18f154e14100e674ed08cb4e1938f0e4177543f439"
dependencies = [
"derive_builder_macro",
"derive_builder_core 0.2.0",
"quote 0.3.15",
"syn 0.11.11",
]
[[package]]
name = "derive_builder_core"
version = "0.12.0"
name = "derive_builder"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0"
dependencies = [
"darling",
"derive_builder_core 0.9.0",
"proc-macro2",
"quote",
"quote 1.0.35",
"syn 1.0.109",
]
[[package]]
name = "derive_builder_macro"
version = "0.12.0"
name = "derive_builder_core"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
checksum = "735e24ee9e5fa8e16b86da5007856e97d592e11867e45d76e0c0d0a164a0b757"
dependencies = [
"derive_builder_core",
"quote 0.3.15",
"syn 0.11.11",
]
[[package]]
name = "derive_builder_core"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef"
dependencies = [
"darling",
"proc-macro2",
"quote 1.0.35",
"syn 1.0.109",
]
@ -249,6 +300,28 @@ dependencies = [
"once_cell",
]
[[package]]
name = "failure"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
dependencies = [
"backtrace",
"failure_derive",
]
[[package]]
name = "failure_derive"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [
"proc-macro2",
"quote 1.0.35",
"syn 1.0.109",
"synstructure",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -428,6 +501,29 @@ dependencies = [
"tracing",
]
[[package]]
name = "iana-time-zone"
version = "0.1.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
@ -542,12 +638,6 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "never"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91"
[[package]]
name = "num-conv"
version = "0.1.0"
@ -616,15 +706,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.55",
"quote 1.0.35",
"syn 2.0.58",
]
[[package]]
name = "pin-project-lite"
version = "0.2.13"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
[[package]]
name = "pin-utils"
@ -649,14 +739,32 @@ dependencies = [
[[package]]
name = "quick-xml"
version = "0.30.0"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956"
checksum = "1d8065cbb01701c11cc195cde85cbf39d1c6a80705b67a157ebb3042e0e5777f"
dependencies = [
"encoding_rs",
"failure",
"log",
"memchr",
]
[[package]]
name = "quick-xml"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd"
dependencies = [
"encoding_rs",
"memchr",
]
[[package]]
name = "quote"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
[[package]]
name = "quote"
version = "1.0.35"
@ -753,14 +861,13 @@ dependencies = [
[[package]]
name = "rss"
version = "2.0.7"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7b2c77eb4450d7d5f98df52c381cd6c4e19b75dad9209a9530b85a44510219a"
checksum = "02e70d6ae72f8a4333af8ce9dce58942020528430eb0d46ee2fcb5e8d4d16377"
dependencies = [
"atom_syndication",
"derive_builder",
"never",
"quick-xml",
"atom_syndication 0.9.1",
"derive_builder 0.9.0",
"quick-xml 0.20.0",
]
[[package]]
@ -771,10 +878,10 @@ dependencies = [
"env_logger",
"log",
"reqwest",
"rss",
"secretfile",
"serde",
"serde_json",
"syndication",
"time",
"tokio",
"toml",
@ -857,8 +964,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.55",
"quote 1.0.35",
"syn 2.0.58",
]
[[package]]
@ -935,9 +1042,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "strsim"
version = "0.10.0"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
[[package]]
name = "subtle"
@ -945,6 +1052,17 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
[[package]]
name = "syn"
version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad"
dependencies = [
"quote 0.3.15",
"synom",
"unicode-xid 0.0.4",
]
[[package]]
name = "syn"
version = "1.0.109"
@ -952,18 +1070,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"quote 1.0.35",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.55"
version = "2.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0"
checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687"
dependencies = [
"proc-macro2",
"quote",
"quote 1.0.35",
"unicode-ident",
]
@ -973,6 +1091,37 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "syndication"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c441f801a54249fe9c74fc03691c6d0f9de2b6ad2f1bf2fb12eb92bd376d3552"
dependencies = [
"atom_syndication 0.6.0",
"rss",
]
[[package]]
name = "synom"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
dependencies = [
"unicode-xid 0.0.4",
]
[[package]]
name = "synstructure"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2",
"quote 1.0.35",
"syn 1.0.109",
"unicode-xid 0.2.4",
]
[[package]]
name = "termcolor"
version = "1.4.1"
@ -998,8 +1147,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.55",
"quote 1.0.35",
"syn 2.0.58",
]
[[package]]
@ -1060,9 +1209,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.36.0"
version = "1.37.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787"
dependencies = [
"backtrace",
"bytes",
@ -1083,8 +1232,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.55",
"quote 1.0.35",
"syn 2.0.58",
]
[[package]]
@ -1229,6 +1378,18 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-xid"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc"
[[package]]
name = "unicode-xid"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "untrusted"
version = "0.9.0"
@ -1287,8 +1448,8 @@ dependencies = [
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.55",
"quote 1.0.35",
"syn 2.0.58",
"wasm-bindgen-shared",
]
@ -1310,7 +1471,7 @@ version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
dependencies = [
"quote",
"quote 1.0.35",
"wasm-bindgen-macro-support",
]
@ -1321,8 +1482,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.55",
"quote 1.0.35",
"syn 2.0.58",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -1383,6 +1544,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
"windows-targets 0.52.4",
]
[[package]]
name = "windows-sys"
version = "0.48.0"

View file

@ -7,9 +7,9 @@ rust-version = "1.67.0"
license = "MIT"
[dependencies]
rss = "2.0.7"
syndication = "0.5.0"
reqwest = { version = "0.12.2", default-features = false, features = ["rustls-tls", "json"] }
tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread", "signal"] }
tokio = { version = "1.37.0", features = ["macros", "rt-multi-thread", "signal"] }
serde = { version = "1.0.197", features = ["derive"] }
serde_json = "1.0.115"
toml = "0.8.8"

View file

@ -7,10 +7,11 @@ use color_eyre::{
Result,
};
use reqwest::Client;
use rss::Channel;
use syndication::Feed;
use std::collections::hash_map::DefaultHasher;
use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use std::str::FromStr;
use tokio::time::sleep;
use tokio::signal::ctrl_c;
use tokio::select;
@ -133,22 +134,32 @@ impl FeedFetcher {
.send()
.await
.wrap_err_with(|| eyre!("Failed to load feed {}", feed))?
.bytes()
.text()
.await
.wrap_err_with(|| eyre!("Failed to load feed {}", feed))?;
let channel = Channel::read_from(content.as_ref())
.wrap_err_with(|| eyre!("Failed to parse feed {}", feed))?;
let item = channel.items.first().ok_or(eyre!("Empty feed"))?;
let channel = Feed::from_str(&content)
.map_err(|_| eyre!("Failed to parse feed {}", feed))?;
let mut hasher = DefaultHasher::new();
if let Some(guid) = item.guid() {
guid.value.hash(&mut hasher);
} else if let Some(date) = item.pub_date() {
date.hash(&mut hasher);
} else if let Some(link) = item.link() {
link.hash(&mut hasher);
} else {
return Err(eyre!("No guid, pubDate or link set on feed item"));
match channel {
Feed::RSS(channel) => {
let item = channel.items.first().ok_or(eyre!("Empty feed"))?;
if let Some(guid) = item.guid() {
guid.value.hash(&mut hasher);
} else if let Some(date) = item.pub_date() {
date.hash(&mut hasher);
} else if let Some(link) = item.link() {
link.hash(&mut hasher);
} else {
return Err(eyre!("No guid, pubDate or link set on feed item"));
}
}
Feed::Atom(channel) => {
let item = channel.entries().first().ok_or(eyre!("Empty feed"))?;
item.id().hash(&mut hasher);
}
}
Ok(hasher.finish())