brmlab/hackerbase
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Preliminary downloading month by month.

Browse source
This commit is contained in:
Dominik Pantůček 2023-07-29 16:04:55 +02:00
parent ec804535c0
commit c17239d3c3
1 changed files with 32 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

56
fetch_fio_fine.sh
View file

@ -313,8 +313,10 @@ merge_acc() {
}
# Current year (last in seq)
# Current year and month
CYEAR=`date +%Y`
CMONTH=`date +%m|sed s/^0//`
MONTHLIMIT=3
# Mark
log "$0" ======== started ========
@ -331,7 +333,8 @@ while read accnt ; do
accrest2=${accrest#* }
YEAR=${accrest2%% *}
# Check starting year before attempting partial downloads
# Check starting year before attempting partial downloads -
# doesn't matter much anymore, but kept as sanity check.
if [ -z "$YEAR" ] ; then
log "Missing start year for account $ACCNO"
continue
@ -345,35 +348,40 @@ while read accnt ; do
continue
fi
# Iterate over years
for year in `seq $YEAR $CYEAR` ; do
# Check whether it needs fetching
CSVNAME="$BANK_DIR_PARTS/$ACCNO-$year.csv"
FILE_DATE=`get_file_date "$CSVNAME"`
NEXT_YEAR=`expr $year + 1`
MIN_DATE="$NEXT_YEAR-01-02"
FILE_TS=`date -d $FILE_DATE +%s`
MIN_TS=`date -d $MIN_DATE +%s`
if [ "$FILE_TS" -lt "$MIN_TS" ] ; then
log "Update $ACCNO in $year - updated $FILE_DATE, needs $MIN_DATE"
download_year "$APIKEY" "$year" "$CSVNAME"
if [ "$year" -lt "$CYEAR" ] ; then
log "Sleeping for 30s before next API usage"
sleep 30
fi
# Iterate over the months
monthsleft=$MONTHLIMIT
currentmonth=$CMONTH
currentyear=$CYEAR
while [ $monthsleft -gt 0 ] ; do
if ! [ "$currentmonth" = "$CMONTH" -a "$currentyear" = "$CYEAR" ] ; then
log "Sleeping for 30s before next API usage"
sleep 30
fi
# Proces unconditionally
currentmonthStr=$currentmonth
if [ $currentmonth -lt 10 ] ; then
currentmonthStr=0$currentmonth
fi
CSVNAME="$BANK_DIR_PARTS/$ACCNO-$currentyear-$currentmonthStr.csv"
log "Update $ACCNO in $currentyear $currentmonth ($currentmonthStr)"
download_year_month "$APIKEY" "$currentyear" "$currentmonthStr" "$CSVNAME"
monthsleft=`expr $monthsleft - 1`
if [ $currentmonth = 1 ] ; then
currentmonth=12
currentyear=`expr $currentyear - 1`
else
log "Skipping $ACCNO in $year - already latest: $FILE_DATE"
currentmonth=`$currentmonth - 1`
fi
done
# Merge the account
log "Merging $ACCNO"
merge_acc "$ACCNO" >"$BANK_DIR/$ACCNO.csv.tmp"
#merge_acc "$ACCNO" >"$BANK_DIR/$ACCNO.csv.tmp"
log "Renaming $ACCNO"
if [ -r "$BANK_DIR/$ACCNO.csv" ] ; then
cp "$BANK_DIR/$ACCNO.csv" "$BANK_DIR/$ACCNO.csv.old"
fi
mv "$BANK_DIR/$ACCNO.csv.tmp" "$BANK_DIR/$ACCNO.csv"
#if [ -r "$BANK_DIR/$ACCNO.csv" ] ; then
#cp "$BANK_DIR/$ACCNO.csv" "$BANK_DIR/$ACCNO.csv.old"
#fi
#mv "$BANK_DIR/$ACCNO.csv.tmp" "$BANK_DIR/$ACCNO.csv"
done < "$APIKEYS_FILE"
# Mark