466 lines
10 KiB
Bash
466 lines
10 KiB
Bash
#!/bin/sh
|
|
#
|
|
# fetch_fio_fine.sh
|
|
#
|
|
# Fio API fine-grained account statements fetcher.
|
|
#
|
|
# ISC License
|
|
#
|
|
# Copyright 2023 Brmlab, z.s.
|
|
# Jan Hrach
|
|
# Dominik Pantůček <dominik.pantucek@trustica.cz>
|
|
#
|
|
# Permission to use, copy, modify, and/or distribute this software
|
|
# for any purpose with or without fee is hereby granted, provided
|
|
# that the above copyright notice and this permission notice appear
|
|
# in all copies.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
|
|
# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
|
|
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
|
|
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
|
|
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
|
|
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
|
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
#
|
|
|
|
# REST API endpoint
|
|
APIURI=https://www.fio.cz/ib_api/rest
|
|
|
|
# Configuration defaults - none
|
|
# TODO: quick fix
|
|
CONFIG_FILE=/home/hackerbase/.hackerbaserc
|
|
CFG_BANK_DIR=
|
|
CFG_APIKEYS_FILE=
|
|
ARG_BANK_DIR=
|
|
ARG_APIKEYS_FILE=
|
|
|
|
# Argument parsing
|
|
while ! [ -z "$1" ] ; do
|
|
case "$1" in
|
|
-apikey)
|
|
ARG_APIKEYS_FILE="$2"
|
|
shift
|
|
shift
|
|
;;
|
|
-bankdir)
|
|
ARG_BANK_DIR="$2"
|
|
shift
|
|
shift
|
|
;;
|
|
-config)
|
|
CONFIG_FILE="$2"
|
|
shift
|
|
shift
|
|
;;
|
|
*)
|
|
echo "Usage: $0 [-config file] [-apikey file] [-bankdir dir]"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Configuration parsing - assumes "dumb" sed which cannot execute
|
|
# multiple statements
|
|
get_config_value() {
|
|
if [ -z "$1" ] ; then
|
|
echo "get_config_value() requires parameter name"
|
|
exit 1
|
|
fi
|
|
if [ -r "$CONFIG_FILE" ] ; then
|
|
cat "$CONFIG_FILE" \
|
|
| sed 's/#.*//' \
|
|
| sed 's/^[ \t]*//' \
|
|
| grep "^$1" \
|
|
| sed 's/^[^ \t]*//' \
|
|
| sed 's/^[ \t]*//' \
|
|
| sed 's/[ \t]*$//'
|
|
fi
|
|
}
|
|
CFG_BANK_DIR=`get_config_value bank-dir`
|
|
CFG_APIKEYS_FILE=`get_config_value apikeys-file`
|
|
|
|
# Configuration merging
|
|
if [ -z "$ARG_BANK_DIR" ] ; then
|
|
BANK_DIR="$CFG_BANK_DIR"
|
|
else
|
|
BANK_DIR="$ARG_BANK_DIR"
|
|
fi
|
|
if [ -z "$ARG_APIKEYS_FILE" ] ; then
|
|
APIKEYS_FILE="$CFG_APIKEYS_FILE"
|
|
else
|
|
APIKEYS_FILE="$ARG_APIKEYS_FILE"
|
|
fi
|
|
|
|
# Storage for partial account statements
|
|
BANK_DIR_PARTS="$BANK_DIR/parts"
|
|
if ! [ -d "$BANK_DIR_PARTS" ] ; then
|
|
mkdir -p "$BANK_DIR_PARTS"
|
|
fi
|
|
|
|
#
|
|
# Very simple "logging" function (stdout should be redirected to log anyway)
|
|
log() {
|
|
echo `date '+%Y-%m-%d %H:%M:%S'` "$@"
|
|
}
|
|
|
|
#
|
|
# Returns the file modification date in YYYY-MM-DD format
|
|
get_file_date() {
|
|
if [ -r "$1" ] ; then
|
|
STAT=`stat -c %y "$1"`
|
|
echo ${STAT%% *}
|
|
else
|
|
echo 2000-01-01
|
|
fi
|
|
}
|
|
|
|
#
|
|
# $1 - URI
|
|
# $2 - output file
|
|
download_file() {
|
|
url="$1"
|
|
fname="$2"
|
|
tmpfname="$fname.tmp"
|
|
oldfname="$fname.old"
|
|
for i in `seq 1 3` ; do
|
|
echo "$url"
|
|
if wget -q "$url" -O "$tmpfname" ; then
|
|
if [ -s "$tmpfname" ] ; then
|
|
log Download OK
|
|
if [ -r "$fname" ] ; then
|
|
cp "$fname" "$oldfname"
|
|
fi
|
|
mv "$tmpfname" "$fname"
|
|
log Rename OK
|
|
break
|
|
else
|
|
log Download successfull but empty or non-existing result.
|
|
log Retrying in 5 s.
|
|
fi
|
|
else
|
|
cat "$tmpfname"
|
|
log Failed download, retrying in 5 s.
|
|
sleep 5
|
|
fi
|
|
done
|
|
}
|
|
|
|
#
|
|
# Leap year?
|
|
leap_year() {
|
|
YEAR="$1"
|
|
ym4=`expr "$YEAR" % 4`
|
|
ym100=`expr "$YEAR" % 100`
|
|
ym400=`expr "$YEAR" % 400`
|
|
if [ $ym4 = 0 ] ; then
|
|
# Maybe
|
|
if ! [ $ym100 = 0 ] ; then
|
|
# Leap
|
|
return 0
|
|
else
|
|
# Maybe
|
|
if [ $ym400 = 0 ] ; then
|
|
# Leap
|
|
return 0
|
|
else
|
|
# Not leap
|
|
return 1
|
|
fi
|
|
fi
|
|
else
|
|
# Not leap
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Returns the last day of given month in Fio format
|
|
# $1 - year
|
|
# $2 - month
|
|
# $3 - if non-empty, return ISO
|
|
end_of_month() {
|
|
YEAR="$1"
|
|
MONTH="$2"
|
|
case $MONTH in
|
|
01) DAY=31 ;;
|
|
02)
|
|
DAY=$(
|
|
if leap_year $YEAR ; then
|
|
echo 29
|
|
else
|
|
echo 28
|
|
fi
|
|
)
|
|
;;
|
|
03) DAY=31 ;;
|
|
04) DAY=30 ;;
|
|
05) DAY=31 ;;
|
|
06) DAY=30 ;;
|
|
07) DAY=31 ;;
|
|
08) DAY=31 ;;
|
|
09) DAY=30 ;;
|
|
10) DAY=31 ;;
|
|
11) DAY=30 ;;
|
|
12) DAY=31 ;;
|
|
esac
|
|
if [ -z "$3" ] ; then
|
|
echo "$DAY.$MONTH.$YEAR"
|
|
else
|
|
echo "$YEAR-$MONTH-$DAY"
|
|
fi
|
|
}
|
|
|
|
#
|
|
# Downloads single year month
|
|
# $1 - apikey
|
|
# $2 - year
|
|
# $3 - month - must be in MM format already
|
|
# $4 - destination file name
|
|
download_year_month() {
|
|
apikey="$1"
|
|
year="$2"
|
|
month="$3"
|
|
fname="$4"
|
|
STARTDATE="$year-$month-01"
|
|
ENDDATE=`end_of_month $year $month 1`
|
|
log "Downloading ${APIURI}/periods/.../$STARTDATE/$ENDDATE/transactions.csv to $fname"
|
|
download_file "${APIURI}/periods/$apikey/$STARTDATE/$ENDDATE/transactions.csv" "$fname"
|
|
}
|
|
|
|
#
|
|
# Gets the first year of the account statement.
|
|
# $1 - account number
|
|
first_acc_part() {
|
|
accno="$1"
|
|
ls "${BANK_DIR_PARTS}/" \
|
|
| grep "^$accno" \
|
|
| grep '[-][0-9][0-9][0-9][0-9].csv$' \
|
|
| sort \
|
|
| head -n 1
|
|
}
|
|
|
|
#
|
|
# Gets the last year of the account statement.
|
|
# $1 - account number
|
|
last_acc_part() {
|
|
accno="$1"
|
|
ls "${BANK_DIR_PARTS}/" \
|
|
| grep "^$accno" \
|
|
| grep '[-][0-9][0-9][0-9][0-9].csv$' \
|
|
| sort -r \
|
|
| head -n 1
|
|
}
|
|
|
|
#
|
|
# Returns all parts except for the first
|
|
# $1 - account number
|
|
all_but_first_parts() {
|
|
accno="$1"
|
|
first="`first_acc_part $1`"
|
|
ls "${BANK_DIR_PARTS}/" \
|
|
| grep "^$accno" \
|
|
| grep '[-][0-9][0-9][0-9][0-9].csv$' \
|
|
| sort \
|
|
| grep -v "$first"
|
|
}
|
|
|
|
#
|
|
# Creates static header for given account
|
|
# $1 - account number
|
|
make_acc_header_static() {
|
|
grep -B 20 '^$' "$1" \
|
|
| grep . \
|
|
| egrep -v 'Balance|^date|^id'
|
|
}
|
|
|
|
#
|
|
# Gets only given header
|
|
# $1 - path to file
|
|
# $2 - header name
|
|
get_header_field() {
|
|
grep "^$2" "$1"
|
|
}
|
|
|
|
#
|
|
# Dynamic part
|
|
# $1 - account number
|
|
make_acc_header() {
|
|
first="`first_acc_part $1`"
|
|
last="`last_acc_part $1`"
|
|
firstfname="$BANK_DIR_PARTS/$first"
|
|
lastfname="$BANK_DIR_PARTS/$last"
|
|
make_acc_header_static "$firstfname"
|
|
get_header_field "$firstfname" openingBalance
|
|
get_header_field "$lastfname" closingBalance
|
|
get_header_field "$firstfname" dateStart
|
|
get_header_field "$lastfname" dateEnd
|
|
get_header_field "$firstfname" idFrom
|
|
get_header_field "$lastfname" idTo
|
|
}
|
|
|
|
#
|
|
# Creates the complete merged account statement from yearly partials
|
|
# $1 - account number
|
|
merge_acc() {
|
|
make_acc_header "$1"
|
|
grep -A 100000 '^$' "$BANK_DIR_PARTS/`first_acc_part $1`"
|
|
for part in `all_but_first_parts $1` ; do
|
|
grep -A 100000 '^ID' "$BANK_DIR_PARTS/$part" \
|
|
| grep -v '^ID'
|
|
done
|
|
}
|
|
|
|
#
|
|
# Gets the first year of the account statement.
|
|
# $1 - account number
|
|
# $2 - year
|
|
first_acc_year_part() {
|
|
accno="$1"
|
|
ls "${BANK_DIR_PARTS}/" \
|
|
| grep "^$accno" \
|
|
| grep "$2-[0-9][0-9].csv$" \
|
|
| sort \
|
|
| head -n 1
|
|
}
|
|
|
|
#
|
|
# Gets the last year of the account statement.
|
|
# $1 - account number
|
|
# $2 - year
|
|
last_acc_year_part() {
|
|
accno="$1"
|
|
ls "${BANK_DIR_PARTS}/" \
|
|
| grep "^$accno" \
|
|
| grep "$2-[0-9][0-9].csv$" \
|
|
| sort -r \
|
|
| head -n 1
|
|
}
|
|
|
|
#
|
|
# Returns all parts except for the first
|
|
# $1 - account number
|
|
# $2 - year
|
|
all_but_first_year_parts() {
|
|
accno="$1"
|
|
first="`first_acc_year_part $1 $2`"
|
|
ls "${BANK_DIR_PARTS}/" \
|
|
| grep "^$accno" \
|
|
| grep "$2-[0-9][0-9].csv$" \
|
|
| sort \
|
|
| grep -v "$first"
|
|
}
|
|
|
|
#
|
|
# Dynamic part
|
|
# $1 - account number
|
|
# $2 - year
|
|
make_acc_year_header() {
|
|
first="`first_acc_year_part $1 $2`"
|
|
last="`last_acc_year_part $1 $2`"
|
|
firstfname="$BANK_DIR_PARTS/$first"
|
|
lastfname="$BANK_DIR_PARTS/$last"
|
|
make_acc_header_static "$firstfname"
|
|
get_header_field "$firstfname" openingBalance
|
|
get_header_field "$lastfname" closingBalance
|
|
get_header_field "$firstfname" dateStart
|
|
get_header_field "$lastfname" dateEnd
|
|
get_header_field "$firstfname" idFrom
|
|
get_header_field "$lastfname" idTo
|
|
}
|
|
|
|
#
|
|
# Single year merged
|
|
# $1 - account number
|
|
# $2 - year
|
|
merge_acc_year() {
|
|
make_acc_year_header "$1" "$2"
|
|
grep -A 100000 '^$' "$BANK_DIR_PARTS/`first_acc_year_part $1 $2`"
|
|
for part in `all_but_first_year_parts $1 $2` ; do
|
|
grep -A 100000 '^ID' "$BANK_DIR_PARTS/$part" \
|
|
| grep -v '^ID'
|
|
done
|
|
}
|
|
|
|
# Current year and month
|
|
CYEAR=`date +%Y`
|
|
CMONTH=`date +%m|sed s/^0//`
|
|
MONTHLIMIT=2
|
|
|
|
# Mark
|
|
log "$0" ======== started ========
|
|
|
|
# Each line should contain account number and Fio API token as first
|
|
# two non-whitespace strings. Third token is the starting year for
|
|
# this account. The rest of each line is ignored. There must be no
|
|
# leading whitespace.
|
|
while read accnt ; do
|
|
# Extract account number, Fio token and starting year
|
|
ACCNO=${accnt%% *}
|
|
accrest=${accnt#* }
|
|
APIKEY=${accrest%% *}
|
|
accrest2=${accrest#* }
|
|
YEAR=${accrest2%% *}
|
|
|
|
# Check starting year before attempting partial downloads -
|
|
# doesn't matter much anymore, but kept as sanity check.
|
|
if [ -z "$YEAR" ] ; then
|
|
log "Missing start year for account $ACCNO"
|
|
continue
|
|
fi
|
|
if [ "$YEAR" -gt "$CYEAR" ] ; then
|
|
log "Start year for account $ACCNO in the future: $YEAR"
|
|
continue
|
|
fi
|
|
if [ "$YEAR" -lt "2010" ] ; then
|
|
log "Start year for account $ACCNO before Brmlab existence: $YEAR"
|
|
continue
|
|
fi
|
|
|
|
# Iterate over the months
|
|
monthsleft=0 # $MONTHLIMIT
|
|
currentmonth=$CMONTH
|
|
currentyear=$CYEAR
|
|
while [ $monthsleft -gt 0 ] ; do
|
|
if ! [ "$currentmonth" = "$CMONTH" -a "$currentyear" = "$CYEAR" ] ; then
|
|
log "Sleeping for 30s before next API usage"
|
|
sleep 30
|
|
fi
|
|
# Proces unconditionally
|
|
currentmonthStr=$currentmonth
|
|
if [ $currentmonth -lt 10 ] ; then
|
|
currentmonthStr=0$currentmonth
|
|
fi
|
|
CSVNAME="$BANK_DIR_PARTS/$ACCNO-$currentyear-$currentmonthStr.csv"
|
|
log "Update $ACCNO in $currentyear $currentmonth ($currentmonthStr)"
|
|
download_year_month "$APIKEY" "$currentyear" "$currentmonthStr" "$CSVNAME"
|
|
monthsleft=`expr $monthsleft - 1`
|
|
if [ $currentmonth = 1 ] ; then
|
|
currentmonth=12
|
|
currentyear=`expr $currentyear - 1`
|
|
else
|
|
currentmonth=`expr $currentmonth - 1`
|
|
fi
|
|
done
|
|
|
|
# Merge the account
|
|
for year in `seq $currentyear $CYEAR` ; do
|
|
log "Merging $ACCNO $year"
|
|
merge_acc_year "$ACCNO" "$year" >"$BANK_DIR_PARTS/$ACCNO-$year.csv.tmp"
|
|
if [ -r "$BANK_DIR_PARTS/$ACCNO-$year.csv" ] ; then
|
|
cp "$BANK_DIR_PARTS/$ACCNO-$year.csv" "$BANK_DIR_PARTS/$ACCNO-$year.csv.old"
|
|
fi
|
|
mv "$BANK_DIR_PARTS/$ACCNO-$year.csv.tmp" "$BANK_DIR_PARTS/$ACCNO-$year.csv"
|
|
done
|
|
log "FINAL Merging $ACCNO"
|
|
merge_acc "$ACCNO" >"$BANK_DIR/$ACCNO.csv.tmp"
|
|
log "Renaming $ACCNO"
|
|
if [ -r "$BANK_DIR/$ACCNO.csv" ] ; then
|
|
cp "$BANK_DIR/$ACCNO.csv" "$BANK_DIR/$ACCNO.csv.old"
|
|
fi
|
|
mv "$BANK_DIR/$ACCNO.csv.tmp" "$BANK_DIR/$ACCNO.csv"
|
|
done < "$APIKEYS_FILE"
|
|
|
|
# Mark
|
|
log "$0" ======== finished ========
|