feat: tweak new readablity lib flags
This commit is contained in:
parent
0424544abf
commit
63705c0745
1 changed files with 21 additions and 2 deletions
23
src/main.rs
23
src/main.rs
|
|
@ -2,6 +2,7 @@ use std::time::Duration;
|
|||
|
||||
use anyhow::{Context, Result};
|
||||
use itertools::Itertools;
|
||||
use readability_rust::{ReadabilityFlags, ReadabilityOptions};
|
||||
use reqwest::{Client, Url};
|
||||
use rss::Channel;
|
||||
use tokio::{task::JoinSet, time::sleep};
|
||||
|
|
@ -58,8 +59,26 @@ async fn complete(mut channel: Channel, client: &Client) -> Result<Box<Channel>>
|
|||
|
||||
async fn get_content(link: &str, client: &Client) -> Result<String> {
|
||||
let response = client.get(link).send().await?;
|
||||
let mut readablity = readability_rust::Readability::new(response.text().await?.as_ref(), None)?;
|
||||
let content = readablity.parse().context("readablity parse error")?.content.context("readablity no content")?;
|
||||
let mut readablity = readability_rust::Readability::new(
|
||||
response.text().await?.as_ref(),
|
||||
Some(ReadabilityOptions {
|
||||
max_elems_to_parse: 0,
|
||||
nb_top_candidates: 5,
|
||||
char_threshold: 500,
|
||||
link_density_modifier: 0.0,
|
||||
flags: ReadabilityFlags {
|
||||
strip_unlikelys: false,
|
||||
clean_conditionally: false,
|
||||
weight_classes: false,
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
)?;
|
||||
let content = readablity
|
||||
.parse()
|
||||
.context("readablity parse error")?
|
||||
.content
|
||||
.context("readablity no content")?;
|
||||
Ok(content)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue