ssdd/hackpsuchatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: switch to chatgpt #4

Merged
ssdd merged 2 commits from chore/switch-to-chatgpt into main 2024-10-13 14:24:07 +00:00
Conversation 0 Commits 2 Files Changed 3 +9
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 60fa93d9e1 - Show all commits

9
src/scrapper/scrape.rs
View File

@ -17,6 +17,7 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
let mut htmls = vec![]; let mut htmls = vec![];
let mut root = handle_inner(&url, runtime).await?; let mut root = handle_inner(&url, runtime).await?;
htmls.push(root.clone()); htmls.push(root.clone());
let mut links = vec![];
loop { loop {
let parsed = Html::parse_document(&root); let parsed = Html::parse_document(&root);
@ -34,6 +35,8 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
match val { match val {
Some(val) => { Some(val) => {
let next = val.value().attr("href").unwrap_or("#"); let next = val.value().attr("href").unwrap_or("#");
links.push(next.to_string());
url = format!("{}{}", url, next); url = format!("{}{}", url, next);
let next = handle_inner(&url, runtime).await?; let next = handle_inner(&url, runtime).await?;
@ -51,6 +54,12 @@ pub async fn handle(mut url: String, runtime: &TargetRuntime) -> anyhow::Result<
.flatten() .flatten()
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let text = mds.join("\n"); let text = mds.join("\n");
let mut text = text.replace("../", "/");
links.into_iter().for_each(|v| {
let v = v.replace("../", "/");
let x = format!("{}{}", url, v);
text = text.replace(&v, &x);
});
Ok(text) Ok(text)
} }