ssdd/hackpsuchatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: ssdd:f030a7cbf8ffd07c1036faf3d600eba0ad6e1cf3
Branches Tags
ssdd:main
..
compare: ssdd:3ccff4a026a80ed321d70b87dae35f5a74a9082e
Branches Tags
ssdd:main

No commits in common. "f030a7cbf8ffd07c1036faf3d600eba0ad6e1cf3" and "3ccff4a026a80ed321d70b87dae35f5a74a9082e" have entirely different histories.
f030a7cbf8 ... 3ccff4a026

1 changed files with 0 additions and 10 deletions
Show Stats Expand all files Collapse all files

10
src/scrapper/scrape.rs
View File

@ -17,7 +17,6 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
let mut htmls = vec![]; let mut htmls = vec![];
let mut root = handle_inner(&url, runtime).await?; let mut root = handle_inner(&url, runtime).await?;
htmls.push(root.clone()); htmls.push(root.clone());
let mut links = vec![];
loop { loop {
let parsed = Html::parse_document(&root); let parsed = Html::parse_document(&root);
@ -35,8 +34,6 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
match val { match val {
Some(val) => { Some(val) => {
let next = val.value().attr("href").unwrap_or("#"); let next = val.value().attr("href").unwrap_or("#");
links.push(next.to_string());
url = format!("{}{}", url, next); url = format!("{}{}", url, next);
let next = handle_inner(&url, runtime).await?; let next = handle_inner(&url, runtime).await?;
@ -54,12 +51,5 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
.flatten() .flatten()
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let text = mds.join("\n"); let text = mds.join("\n");
let mut text = text.replace("../", "/");
links.into_iter().for_each(|v| {
let v = v.replace("../", "/");
let x = format!("{}{}", url, v);
text = text.replace(&v, &x);
});
Ok(text) Ok(text)
} }