ssdd/hackpsuchatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'fix: invalid url' (#3) from fix/invalid-url into main

Browse Source 
Reviewed-on: #3
This commit is contained in:
sapan 2024-10-13 06:52:24 +00:00
commit f030a7cbf8
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/scrapper/scrape.rs
View File

@ -17,6 +17,7 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
let mut htmls = vec![];
let mut root = handle_inner(&url, runtime).await?;
htmls.push(root.clone());
let mut links = vec![];
loop {
let parsed = Html::parse_document(&root);
@ -34,6 +35,8 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
match val {
Some(val) => {
let next = val.value().attr("href").unwrap_or("#");
links.push(next.to_string());
url = format!("{}{}", url, next);
let next = handle_inner(&url, runtime).await?;
@ -51,5 +54,12 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
.flatten()
.collect::<Vec<_>>();
let text = mds.join("\n");
let mut text = text.replace("../", "/");
links.into_iter().for_each(|v| {
let v = v.replace("../", "/");
let x = format!("{}{}", url, v);
text = text.replace(&v, &x);
});
Ok(text)
}