From 0882cafb03dd9a5fc5ddbd9a4e0a5be2e092fca0 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Fri, 10 Jul 2020 22:04:09 +0200 Subject: fetch-pages: add retry --- fetch-pages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-pages b/fetch-pages index be2209a..6cbc60b 100755 --- a/fetch-pages +++ b/fetch-pages @@ -35,7 +35,7 @@ while [ "$url" ]; do if [ "$since" -le "$LIMIT" -a "$since" -ne 0 ]; then break; fi echo "Fetching ${BASE}${url}..." >&2 - curl -f -L -o "${OUTDIR}/pages/${since}.html" "${BASE}${url}" + curl -f --retry 20 -L -o "${OUTDIR}/pages/${since}.html" "${BASE}${url}" url="$(sed -r -n '/SOUP\.Endless\.next_url/ {s#^.*SOUP\.Endless\.next_url = '\''([^'\'']+)'\'';.*$#\1#;p}' "${OUTDIR}/pages/${since}.html")" # Be nice, don't overload the servers! -- cgit v1.2.3