summaryrefslogtreecommitdiffstats
path: root/fetch-pages
diff options
context:
space:
mode:
authorMatthias Schiffer <mschiffer@universe-factory.net>2020-07-10 22:04:09 +0200
committerMatthias Schiffer <mschiffer@universe-factory.net>2020-07-10 22:04:09 +0200
commit0882cafb03dd9a5fc5ddbd9a4e0a5be2e092fca0 (patch)
treef61c5eeb9c2781458a0c6d33d200436e8af4975d /fetch-pages
parent765ed07354c655210ee25586988bad98353ebeef (diff)
downloadsoup-backup-0882cafb03dd9a5fc5ddbd9a4e0a5be2e092fca0.tar
soup-backup-0882cafb03dd9a5fc5ddbd9a4e0a5be2e092fca0.zip
fetch-pages: add retry
Diffstat (limited to 'fetch-pages')
-rwxr-xr-xfetch-pages2
1 files changed, 1 insertions, 1 deletions
diff --git a/fetch-pages b/fetch-pages
index be2209a..6cbc60b 100755
--- a/fetch-pages
+++ b/fetch-pages
@@ -35,7 +35,7 @@ while [ "$url" ]; do
if [ "$since" -le "$LIMIT" -a "$since" -ne 0 ]; then break; fi
echo "Fetching ${BASE}${url}..." >&2
- curl -f -L -o "${OUTDIR}/pages/${since}.html" "${BASE}${url}"
+ curl -f --retry 20 -L -o "${OUTDIR}/pages/${since}.html" "${BASE}${url}"
url="$(sed -r -n '/SOUP\.Endless\.next_url/ {s#^.*SOUP\.Endless\.next_url = '\''([^'\'']+)'\'';.*$#\1#;p}' "${OUTDIR}/pages/${since}.html")"
# Be nice, don't overload the servers!