Batch large requests into batches of 1000

saca-sapos

@@ -25,7 +25,7 @@ get_config() {
   curl -s "$SERVER"
 }
 
-indexer() {
+download_many() {
   AMOUNT="$1"
   export SAPO_CONTACT="$2"
 
@@ -39,20 +39,24 @@ indexer() {
     exit 1;
   fi
 
-  VIDEOS="$(curl -s "$SAPO_CONFIG_VIDEOS_URL?amount=$AMOUNT")"
-
-  if [ "$(echo "$VIDEOS" | jq -r .error)" = "greedy" ]; then
-    echo "Não podes pedir mais do que $SAPO_CONFIG_MAX vídeos!" >&2
-    exit 1;
-  fi
-
   mkdir -p videos
-  echo "$VIDEOS" |
+  while [ "$AMOUNT" -gt 0 ]; do
-    jq -r '.videos[]' |
+    if [ "$AMOUNT" -ge 1000 ]; then
-    xargs "-P$PROCS" -I {} "$SAPO_EXEC" single {}
+      N=1000
+    else
+      N=$AMOUNT
+    fi
+
+    echo "another part ($N)" >&2
+    curl -s "$SAPO_CONFIG_VIDEOS_URL?amount=$N" |
+      jq -r '.videos[]' |
+      xargs "-P$PROCS" -I {} "$SAPO_EXEC" single {}
+
+    AMOUNT=$(( $AMOUNT - $N ))
+  done
 }
 
-download() {
+download_single() {
   VIDEO_ID="$1"
   VIDEO_URL="https://rd.videos.sapo.pt/$VIDEO_ID/mov/1"
   FILENAME="videos/$VIDEO_ID.mp4"
@@ -96,10 +100,10 @@ unknown() {
 
 case "$1" in
   "download")
-    indexer "$2" "$3"
+    download_many "$2" "$3"
     ;;
   "single")
-    download "$2"
+    download_single "$2"
     ;;
   "unknown")
     unknown