aboutsummaryrefslogtreecommitdiffstats
path: root/Information gathering 2/Resources/crawl.bash
blob: 13f729c357623fa20506c3d5ed7e0ca9d21ac629 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/usr/bin/bash

get_next_step() {
	url="$1"
	curl --follow --silent "$url" | grep '^<a href=' | grep --only-matching '"[^"]*"' | tr --delete \"
}

urls=(http://10.0.2.15/.hidden/)
while ((${#urls[@]} > 0)); do
	current="${urls[0]}"
	urls=("${urls[@]:1}")

	if [[ "${current[i]: -1}" == / ]]; then
		mapfile -t next_steps < <(get_next_step "$current")
		urls+=("${next_steps[@]/#/$current}")
	else
		echo -en "$current\t"
		curl --silent "$current"
	fi
done