Create new fio fetcher as clone of the original.
This commit is contained in:
parent
053729fe7d
commit
41b218613a
1 changed files with 311 additions and 0 deletions
311
fetch_fio_fine.sh
Normal file
311
fetch_fio_fine.sh
Normal file
|
|
@ -0,0 +1,311 @@
|
||||||
|
#!/bin/sh
|
||||||
|
#
|
||||||
|
# fetch_fio.sh
|
||||||
|
#
|
||||||
|
# Fio API account statements fetcher.
|
||||||
|
#
|
||||||
|
# ISC License
|
||||||
|
#
|
||||||
|
# Copyright 2023 Brmlab, z.s.
|
||||||
|
# Jan Hrach
|
||||||
|
# Dominik Pantůček <dominik.pantucek@trustica.cz>
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and/or distribute this software
|
||||||
|
# for any purpose with or without fee is hereby granted, provided
|
||||||
|
# that the above copyright notice and this permission notice appear
|
||||||
|
# in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
|
||||||
|
# WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
|
||||||
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
|
||||||
|
# AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
|
||||||
|
# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
|
||||||
|
# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
|
||||||
|
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
||||||
|
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
|
||||||
|
# REST API endpoint
|
||||||
|
APIURI=https://www.fio.cz/ib_api/rest
|
||||||
|
|
||||||
|
# Configuration defaults - none
|
||||||
|
# TODO: quick fix
|
||||||
|
CONFIG_FILE=/home/hackerbase/.hackerbaserc
|
||||||
|
CFG_BANK_DIR=
|
||||||
|
CFG_APIKEYS_FILE=
|
||||||
|
ARG_BANK_DIR=
|
||||||
|
ARG_APIKEYS_FILE=
|
||||||
|
|
||||||
|
# Argument parsing
|
||||||
|
while ! [ -z "$1" ] ; do
|
||||||
|
case "$1" in
|
||||||
|
-apikey)
|
||||||
|
ARG_APIKEYS_FILE="$2"
|
||||||
|
shift
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-bankdir)
|
||||||
|
ARG_BANK_DIR="$2"
|
||||||
|
shift
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-config)
|
||||||
|
CONFIG_FILE="$2"
|
||||||
|
shift
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Usage: $0 [-config file] [-apikey file] [-bankdir dir]"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Configuration parsing - assumes "dumb" sed which cannot execute
|
||||||
|
# multiple statements
|
||||||
|
get_config_value() {
|
||||||
|
if [ -z "$1" ] ; then
|
||||||
|
echo "get_config_value() requires parameter name"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -r "$CONFIG_FILE" ] ; then
|
||||||
|
cat "$CONFIG_FILE" \
|
||||||
|
| sed 's/#.*//' \
|
||||||
|
| sed 's/^[ \t]*//' \
|
||||||
|
| grep "^$1" \
|
||||||
|
| sed 's/^[^ \t]*//' \
|
||||||
|
| sed 's/^[ \t]*//' \
|
||||||
|
| sed 's/[ \t]*$//'
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
CFG_BANK_DIR=`get_config_value bank-dir`
|
||||||
|
CFG_APIKEYS_FILE=`get_config_value apikeys-file`
|
||||||
|
|
||||||
|
# Configuration merging
|
||||||
|
if [ -z "$ARG_BANK_DIR" ] ; then
|
||||||
|
BANK_DIR="$CFG_BANK_DIR"
|
||||||
|
else
|
||||||
|
BANK_DIR="$ARG_BANK_DIR"
|
||||||
|
fi
|
||||||
|
if [ -z "$ARG_APIKEYS_FILE" ] ; then
|
||||||
|
APIKEYS_FILE="$CFG_APIKEYS_FILE"
|
||||||
|
else
|
||||||
|
APIKEYS_FILE="$ARG_APIKEYS_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Storage for partial account statements
|
||||||
|
BANK_DIR_PARTS="$BANK_DIR/parts"
|
||||||
|
if ! [ -d "$BANK_DIR_PARTS" ] ; then
|
||||||
|
mkdir -p "$BANK_DIR_PARTS"
|
||||||
|
fi
|
||||||
|
|
||||||
|
#
|
||||||
|
# Very simple "logging" function (stdout should be redirected to log anyway)
|
||||||
|
log() {
|
||||||
|
echo `date '+%Y-%m-%d %H:%M:%S'` "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Returns the file modification date in YYYY-MM-DD format
|
||||||
|
get_file_date() {
|
||||||
|
if [ -r "$1" ] ; then
|
||||||
|
STAT=`stat -c %y "$1"`
|
||||||
|
echo ${STAT%% *}
|
||||||
|
else
|
||||||
|
echo 2000-01-01
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# $1 - URI
|
||||||
|
# $2 - output file
|
||||||
|
download_file() {
|
||||||
|
url="$1"
|
||||||
|
fname="$2"
|
||||||
|
tmpfname="$fname.tmp"
|
||||||
|
oldfname="$fname.old"
|
||||||
|
for i in `seq 1 3` ; do
|
||||||
|
echo "$url"
|
||||||
|
if wget -q "$url" -O "$tmpfname" ; then
|
||||||
|
if [ -s "$tmpfname" ] ; then
|
||||||
|
log Download OK
|
||||||
|
if [ -r "$fname" ] ; then
|
||||||
|
cp "$fname" "$oldfname"
|
||||||
|
fi
|
||||||
|
mv "$tmpfname" "$fname"
|
||||||
|
log Rename OK
|
||||||
|
break
|
||||||
|
else
|
||||||
|
log Download successfull but empty or non-existing result.
|
||||||
|
log Retrying in 5 s.
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
cat "$tmpfname"
|
||||||
|
log Failed download, retrying in 5 s.
|
||||||
|
sleep 5
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Downloads single year
|
||||||
|
# $1 - apikey
|
||||||
|
# $2 - year
|
||||||
|
# $3 - destination file name
|
||||||
|
download_year() {
|
||||||
|
apikey="$1"
|
||||||
|
year="$2"
|
||||||
|
fname="$3"
|
||||||
|
log "Downloading ${APIURI}/periods/.../$year-01-01/$year-12-31/transactions.csv to $fname"
|
||||||
|
download_file "${APIURI}/periods/$apikey/$year-01-01/$year-12-31/transactions.csv" "$fname"
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Gets the first year of the account statement.
|
||||||
|
# $1 - account number
|
||||||
|
first_acc_part() {
|
||||||
|
accno="$1"
|
||||||
|
ls "${BANK_DIR_PARTS}/" \
|
||||||
|
| grep "^$accno" \
|
||||||
|
| grep '.csv$' \
|
||||||
|
| sort \
|
||||||
|
| head -n 1
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Gets the last year of the account statement.
|
||||||
|
# $1 - account number
|
||||||
|
last_acc_part() {
|
||||||
|
accno="$1"
|
||||||
|
ls "${BANK_DIR_PARTS}/" \
|
||||||
|
| grep "^$accno" \
|
||||||
|
| grep '.csv$' \
|
||||||
|
| sort -r \
|
||||||
|
| head -n 1
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Returns all parts except for the first
|
||||||
|
# $1 - account number
|
||||||
|
all_but_first_parts() {
|
||||||
|
accno="$1"
|
||||||
|
first="`first_acc_part $1`"
|
||||||
|
ls "${BANK_DIR_PARTS}/" \
|
||||||
|
| grep "^$accno" \
|
||||||
|
| grep '.csv$' \
|
||||||
|
| sort \
|
||||||
|
| grep -v "$first"
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Creates static header for given account
|
||||||
|
# $1 - account number
|
||||||
|
make_acc_header_static() {
|
||||||
|
grep -B 20 '^$' "$1" \
|
||||||
|
| grep . \
|
||||||
|
| egrep -v 'Balance|^date|^id'
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Gets only given header
|
||||||
|
# $1 - path to file
|
||||||
|
# $2 - header name
|
||||||
|
get_header_field() {
|
||||||
|
grep "^$2" "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Dynamic part
|
||||||
|
# $1 - account number
|
||||||
|
make_acc_header() {
|
||||||
|
first="`first_acc_part $1`"
|
||||||
|
last="`last_acc_part $2`"
|
||||||
|
firstfname="$BANK_DIR_PARTS/$first"
|
||||||
|
lastfname="$BANK_DIR_PARTS/$last"
|
||||||
|
make_acc_header_static "$firstfname"
|
||||||
|
get_header_field "$firstfname" openingBalance
|
||||||
|
get_header_field "$lastfname" closingBalance
|
||||||
|
get_header_field "$firstfname" dateStart
|
||||||
|
get_header_field "$lastfname" dateEnd
|
||||||
|
get_header_field "$firstfname" idFrom
|
||||||
|
get_header_field "$lastfname" idTo
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# Creates the complete merged account statement
|
||||||
|
# $1 - account number
|
||||||
|
merge_acc() {
|
||||||
|
make_acc_header "$1"
|
||||||
|
grep -A 100000 '^$' "$BANK_DIR_PARTS/`first_acc_part $1`"
|
||||||
|
for part in `all_but_first_parts $1` ; do
|
||||||
|
grep -A 100000 '^ID' "$BANK_DIR_PARTS/$part" \
|
||||||
|
| grep -v '^ID'
|
||||||
|
done
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
# Current year (last in seq)
|
||||||
|
CYEAR=`date +%Y`
|
||||||
|
|
||||||
|
# Mark
|
||||||
|
log "$0" ======== started ========
|
||||||
|
|
||||||
|
# Each line should contain account number and Fio API token as first
|
||||||
|
# two non-whitespace strings. Third token is the starting year for
|
||||||
|
# this account. The rest of each line is ignored. There must be no
|
||||||
|
# leading whitespace.
|
||||||
|
while read accnt ; do
|
||||||
|
# Extract account number, Fio token and starting year
|
||||||
|
ACCNO=${accnt%% *}
|
||||||
|
accrest=${accnt#* }
|
||||||
|
APIKEY=${accrest%% *}
|
||||||
|
accrest2=${accrest#* }
|
||||||
|
YEAR=${accrest2%% *}
|
||||||
|
|
||||||
|
# Check starting year before attempting partial downloads
|
||||||
|
if [ -z "$YEAR" ] ; then
|
||||||
|
log "Missing start year for account $ACCNO"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [ "$YEAR" -gt "$CYEAR" ] ; then
|
||||||
|
log "Start year for account $ACCNO in the future: $YEAR"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [ "$YEAR" -lt "2010" ] ; then
|
||||||
|
log "Start year for account $ACCNO before Brmlab existence: $YEAR"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Iterate over years
|
||||||
|
for year in `seq $YEAR $CYEAR` ; do
|
||||||
|
# Check whether it needs fetching
|
||||||
|
CSVNAME="$BANK_DIR_PARTS/$ACCNO-$year.csv"
|
||||||
|
FILE_DATE=`get_file_date "$CSVNAME"`
|
||||||
|
NEXT_YEAR=`expr $year + 1`
|
||||||
|
MIN_DATE="$NEXT_YEAR-01-02"
|
||||||
|
FILE_TS=`date -d $FILE_DATE +%s`
|
||||||
|
MIN_TS=`date -d $MIN_DATE +%s`
|
||||||
|
if [ "$FILE_TS" -lt "$MIN_TS" ] ; then
|
||||||
|
log "Update $ACCNO in $year - updated $FILE_DATE, needs $MIN_DATE"
|
||||||
|
download_year "$APIKEY" "$year" "$CSVNAME"
|
||||||
|
if [ "$year" -lt "$CYEAR" ] ; then
|
||||||
|
log "Sleeping for 30s before next API usage"
|
||||||
|
sleep 30
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log "Skipping $ACCNO in $year - already latest: $FILE_DATE"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Merge the account
|
||||||
|
log "Merging $ACCNO"
|
||||||
|
merge_acc "$ACCNO" >"$BANK_DIR/$ACCNO.csv.tmp"
|
||||||
|
log "Renaming $ACCNO"
|
||||||
|
if [ -r "$BANK_DIR/$ACCNO.csv" ] ; then
|
||||||
|
cp "$BANK_DIR/$ACCNO.csv" "$BANK_DIR/$ACCNO.csv.old"
|
||||||
|
fi
|
||||||
|
mv "$BANK_DIR/$ACCNO.csv.tmp" "$BANK_DIR/$ACCNO.csv"
|
||||||
|
done < "$APIKEYS_FILE"
|
||||||
|
|
||||||
|
# Mark
|
||||||
|
log "$0" ======== finished ========
|
||||||
Loading…
Add table
Add a link
Reference in a new issue