brmlab/hackerbase
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Preliminary downloading month by month.

Browse source
This commit is contained in:
Dominik Pantůček 2023-07-29 16:04:55 +02:00
parent ec804535c0
commit c17239d3c3
1 changed files with 32 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

56
fetch_fio_fine.sh
View file

@ -313,8 +313,10 @@ merge_acc() {
} }
# Current year (last in seq) # Current year and month
CYEAR=`date +%Y` CYEAR=`date +%Y`
CMONTH=`date +%m|sed s/^0//`
MONTHLIMIT=3
# Mark # Mark
log "$0" ======== started ======== log "$0" ======== started ========
@ -331,7 +333,8 @@ while read accnt ; do
accrest2=${accrest#* } accrest2=${accrest#* }
YEAR=${accrest2%% *} YEAR=${accrest2%% *}
# Check starting year before attempting partial downloads # Check starting year before attempting partial downloads -
# doesn't matter much anymore, but kept as sanity check.
if [ -z "$YEAR" ] ; then if [ -z "$YEAR" ] ; then
log "Missing start year for account $ACCNO" log "Missing start year for account $ACCNO"
continue continue
@ -345,35 +348,40 @@ while read accnt ; do
continue continue
fi fi
# Iterate over years # Iterate over the months
for year in `seq $YEAR $CYEAR` ; do monthsleft=$MONTHLIMIT
# Check whether it needs fetching currentmonth=$CMONTH
CSVNAME="$BANK_DIR_PARTS/$ACCNO-$year.csv" currentyear=$CYEAR
FILE_DATE=`get_file_date "$CSVNAME"` while [ $monthsleft -gt 0 ] ; do
NEXT_YEAR=`expr $year + 1` if ! [ "$currentmonth" = "$CMONTH" -a "$currentyear" = "$CYEAR" ] ; then
MIN_DATE="$NEXT_YEAR-01-02" log "Sleeping for 30s before next API usage"
FILE_TS=`date -d $FILE_DATE +%s` sleep 30
MIN_TS=`date -d $MIN_DATE +%s` fi
if [ "$FILE_TS" -lt "$MIN_TS" ] ; then # Proces unconditionally
log "Update $ACCNO in $year - updated $FILE_DATE, needs $MIN_DATE" currentmonthStr=$currentmonth
download_year "$APIKEY" "$year" "$CSVNAME" if [ $currentmonth -lt 10 ] ; then
if [ "$year" -lt "$CYEAR" ] ; then currentmonthStr=0$currentmonth
log "Sleeping for 30s before next API usage" fi
sleep 30 CSVNAME="$BANK_DIR_PARTS/$ACCNO-$currentyear-$currentmonthStr.csv"
fi log "Update $ACCNO in $currentyear $currentmonth ($currentmonthStr)"
download_year_month "$APIKEY" "$currentyear" "$currentmonthStr" "$CSVNAME"
monthsleft=`expr $monthsleft - 1`
if [ $currentmonth = 1 ] ; then
currentmonth=12
currentyear=`expr $currentyear - 1`
else else
log "Skipping $ACCNO in $year - already latest: $FILE_DATE" currentmonth=`$currentmonth - 1`
fi fi
done done
# Merge the account # Merge the account
log "Merging $ACCNO" log "Merging $ACCNO"
merge_acc "$ACCNO" >"$BANK_DIR/$ACCNO.csv.tmp" #merge_acc "$ACCNO" >"$BANK_DIR/$ACCNO.csv.tmp"
log "Renaming $ACCNO" log "Renaming $ACCNO"
if [ -r "$BANK_DIR/$ACCNO.csv" ] ; then #if [ -r "$BANK_DIR/$ACCNO.csv" ] ; then
cp "$BANK_DIR/$ACCNO.csv" "$BANK_DIR/$ACCNO.csv.old" #cp "$BANK_DIR/$ACCNO.csv" "$BANK_DIR/$ACCNO.csv.old"
fi #fi
mv "$BANK_DIR/$ACCNO.csv.tmp" "$BANK_DIR/$ACCNO.csv" #mv "$BANK_DIR/$ACCNO.csv.tmp" "$BANK_DIR/$ACCNO.csv"
done < "$APIKEYS_FILE" done < "$APIKEYS_FILE"
# Mark # Mark