ssdd/hackpsuchatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: invalid url

Browse Source
This commit is contained in:
Sapan Shah 2024-10-13 12:18:03 +05:30
parent 3ccff4a026
commit edf657157a
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/scrapper/scrape.rs
View File

@ -17,6 +17,7 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
let mut htmls = vec![]; let mut htmls = vec![];
let mut root = handle_inner(&url, runtime).await?; let mut root = handle_inner(&url, runtime).await?;
htmls.push(root.clone()); htmls.push(root.clone());
let mut links = vec![];
loop { loop {
let parsed = Html::parse_document(&root); let parsed = Html::parse_document(&root);
@ -34,6 +35,8 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
match val { match val {
Some(val) => { Some(val) => {
let next = val.value().attr("href").unwrap_or("#"); let next = val.value().attr("href").unwrap_or("#");
links.push(next.to_string());
url = format!("{}{}", url, next); url = format!("{}{}", url, next);
let next = handle_inner(&url, runtime).await?; let next = handle_inner(&url, runtime).await?;
@ -51,5 +54,12 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
.flatten() .flatten()
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let text = mds.join("\n"); let text = mds.join("\n");
let mut text = text.replace("../", "/");
links.into_iter().for_each(|v| {
let v = v.replace("../", "/");
let x = format!("{}{}", url, v);
text = text.replace(&v, &x);
});
Ok(text) Ok(text)
} }