Work on partial downloads.

fetch_fio.sh

@@ -107,8 +107,12 @@ log() {
 #
 # Returns the file modification date in YYYY-MM-DD format
 get_file_date() {
-    STAT=`stat -c %y "$1"`
-    echo ${STAT%% *}
+    if [ -r "$1" ] ; then
+	STAT=`stat -c %y "$1"`
+	echo ${STAT%% *}
+    else
+	echo 2000-01-01
+    fi
 }
 
 #
@@ -124,7 +128,9 @@ download_file() {
 	if wget -q "$url" -O "$tmpfname" ; then
 	    if [ -s  "$tmpfname" ] ; then
 		log Download OK
-		cp "$fname" "$oldfname"
+		if [ -r "$fname" ] ; then
+		    cp "$fname" "$oldfname"
+		fi
 		mv "$tmpfname" "$fname"
 		log Finished
 		break
@@ -145,7 +151,7 @@ download_file() {
 # $2 - year
 # $3 - destination file name
 download_year() {
-    echo "${APIURI}/periods/$1/$2-01-01/$2-12-31/transactions.csv"
+    download_file "${APIURI}/periods/$1/$2-01-01/$2-12-31/transactions.csv" "$3"
 }
 
 # Current year (last in seq)
@@ -182,13 +188,23 @@ while read accnt ; do
     
     # Iterate over years
     for year in `seq $YEAR $CYEAR` ; do
-	# Log action
-	log "Processing account $ACCNO for $year"
-
 	# Check whether it needs fetching
 	CSVNAME="$BANK_DIR_PARTS/$ACCNO-$year.csv"
-	echo "CSVNAME=$CSVNAME"
-	download_year "$APIKEY" $year $CSVNAME
+	UPDATED=`get_file_date "$CSVNAME"`
+	NYEAR=`expr $year + 1`
+	MINUPD="$NYEAR-01-02"
+	UTS=`date -d $UPDATED +%s`
+	MTS=`date -d $MINUPD +%s`
+	if [ "$UTS" -lt "$MTS" ] ; then
+	    log "Update $ACCNO in $year - updated $UPDATED, needs $MINUPD"
+	    download_year "$APIKEY" "$year" "$CSVNAME"
+	    if [ "$year" -lt "$CYEAR" ] ; then
+		log "Sleeping for 30s before next API usage"
+		sleep 30
+	    fi
+	else
+	    log "Skipping $ACCNO in $year - already latest"
+	fi
     done
 done < "$APIKEYS_FILE"