Batch large requests into batches of 1000

This commit is contained in:
Hugo Peixoto 2023-08-01 02:46:35 +01:00
parent a55294e7e4
commit 53fc8b74bb

View File

@ -25,7 +25,7 @@ get_config() {
curl -s "$SERVER"
}
indexer() {
download_many() {
AMOUNT="$1"
export SAPO_CONTACT="$2"
@ -39,20 +39,24 @@ indexer() {
exit 1;
fi
VIDEOS="$(curl -s "$SAPO_CONFIG_VIDEOS_URL?amount=$AMOUNT")"
if [ "$(echo "$VIDEOS" | jq -r .error)" = "greedy" ]; then
echo "Não podes pedir mais do que $SAPO_CONFIG_MAX vídeos!" >&2
exit 1;
mkdir -p videos
while [ "$AMOUNT" -gt 0 ]; do
if [ "$AMOUNT" -ge 1000 ]; then
N=1000
else
N=$AMOUNT
fi
mkdir -p videos
echo "$VIDEOS" |
echo "another part ($N)" >&2
curl -s "$SAPO_CONFIG_VIDEOS_URL?amount=$N" |
jq -r '.videos[]' |
xargs "-P$PROCS" -I {} "$SAPO_EXEC" single {}
AMOUNT=$(( $AMOUNT - $N ))
done
}
download() {
download_single() {
VIDEO_ID="$1"
VIDEO_URL="https://rd.videos.sapo.pt/$VIDEO_ID/mov/1"
FILENAME="videos/$VIDEO_ID.mp4"
@ -96,10 +100,10 @@ unknown() {
case "$1" in
"download")
indexer "$2" "$3"
download_many "$2" "$3"
;;
"single")
download "$2"
download_single "$2"
;;
"unknown")
unknown