brmlab/hackerbase
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Work on partial downloads.

Browse source
This commit is contained in:
Dominik Pantůček 2023-05-25 23:35:06 +02:00
parent d5c016e4d4
commit 5287c17d19
1 changed files with 25 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

28
fetch_fio.sh
View file

@ -107,8 +107,12 @@ log() {
# #
# Returns the file modification date in YYYY-MM-DD format # Returns the file modification date in YYYY-MM-DD format
get_file_date() { get_file_date() {
if [ -r "$1" ] ; then
STAT=`stat -c %y "$1"` STAT=`stat -c %y "$1"`
echo ${STAT%% *} echo ${STAT%% *}
else
echo 2000-01-01
fi
} }
# #
@ -124,7 +128,9 @@ download_file() {
if wget -q "$url" -O "$tmpfname" ; then if wget -q "$url" -O "$tmpfname" ; then
if [ -s "$tmpfname" ] ; then if [ -s "$tmpfname" ] ; then
log Download OK log Download OK
if [ -r "$fname" ] ; then
cp "$fname" "$oldfname" cp "$fname" "$oldfname"
fi
mv "$tmpfname" "$fname" mv "$tmpfname" "$fname"
log Finished log Finished
break break
@ -145,7 +151,7 @@ download_file() {
# $2 - year # $2 - year
# $3 - destination file name # $3 - destination file name
download_year() { download_year() {
echo "${APIURI}/periods/$1/$2-01-01/$2-12-31/transactions.csv" download_file "${APIURI}/periods/$1/$2-01-01/$2-12-31/transactions.csv" "$3"
} }
# Current year (last in seq) # Current year (last in seq)
@ -182,13 +188,23 @@ while read accnt ; do
# Iterate over years # Iterate over years
for year in `seq $YEAR $CYEAR` ; do for year in `seq $YEAR $CYEAR` ; do
# Log action
log "Processing account $ACCNO for $year"
# Check whether it needs fetching # Check whether it needs fetching
CSVNAME="$BANK_DIR_PARTS/$ACCNO-$year.csv" CSVNAME="$BANK_DIR_PARTS/$ACCNO-$year.csv"
echo "CSVNAME=$CSVNAME" UPDATED=`get_file_date "$CSVNAME"`
download_year "$APIKEY" $year $CSVNAME NYEAR=`expr $year + 1`
MINUPD="$NYEAR-01-02"
UTS=`date -d $UPDATED +%s`
MTS=`date -d $MINUPD +%s`
if [ "$UTS" -lt "$MTS" ] ; then
log "Update $ACCNO in $year - updated $UPDATED, needs $MINUPD"
download_year "$APIKEY" "$year" "$CSVNAME"
if [ "$year" -lt "$CYEAR" ] ; then
log "Sleeping for 30s before next API usage"
sleep 30
fi
else
log "Skipping $ACCNO in $year - already latest"
fi
done done
done < "$APIKEYS_FILE" done < "$APIKEYS_FILE"