diff --git a/update.py b/update.py index f18f48a..9b7e5c4 100755 --- a/update.py +++ b/update.py @@ -1,74 +1,75 @@ -#!/usr/bin/python +#!/usr/bin/env python import csv -import lzma +from backports import lzma import sys import os from datetime import date n_returns = 600 -sensorlist = [1,2,3,4,5,6,7,8,9,11,12,13,14,15,16,17,18,19,20] +sensorlist = [1,2,4,8,11,12,13,14,15,16,17,18,19,20] + +frm_s = sys.argv[1] +to_s = sys.argv[2] +#div = sys.argv[3] + +frm_y, frm_m, frm_d = [int(x) for x in frm_s.split('/')] +to_y, to_m, to_d = [int(x) for x in to_s.split('/')] -frm = sys.argv[1] -to = sys.argv[2] -div = sys.argv[3] - -frm_y, frm_m, frm_d = [int(x) for x in frm.split('/')] -to_y, to_m, to_d = [int(x) for x in to.split('/')] - -frm = date(frm_y, frm_m, frm_d) # align for naughty input -to = date(to_y, to_m, to_d) # align for naughty input - -# retreive from database -# first, calculate culling idx +# first, calculate culling step +frm = date(frm_y, frm_m, frm_d) +to = date(to_y, to_m, to_d) +frm_s = frm.strftime("%Y/%m/%d 00:00") # refine input +to_s = to.strftime("%Y/%m/%d 23:59") # refine input samples_per_day = 24 * 6 ndays = (to - frm).days * samples_per_day step = int(ndays / n_returns) + 1 +# retreive from database result = [] while frm.year < to.year or frm.month <= to.month: - tmplist = [] - for i in sensorlist: - no = str(i) - fname = "/home/tutimura/public_html/cgi-bin/ondotori/data/"+ no+ - "/"+frm.strftime("%Y%m")+".csv.xz" - try: - xz = lzma.LZMAFile(fname) - r1 = csv.DictReader(xz, fieldnames=["datetime", "temp-"+no, "humid-"+no]) - frm_s = frm.strftime("%Y/%m/%d 00:00") - to_s = to.strftime("%Y/%m/%d 00:00") - tmp=filter(lambda x: x["datetime"] >= frm.strftime("%Y/%m/%d") + " 00:00" - and x["datetime"] <= to + " 23:59", r1) - if tmp == []: continue - tmplist.append(tmp[::step]) - except IOError: - break - result += [x for s in zip(*tmplist) for x in s] + tmplist = [] - frm_m += 1 - if frm_m > 12: - frm_y += 1 - frm_m = 1 + for i in sensorlist: + no = str(i) + fname = "data/"+ no+ frm.strftime("/%Y%m.csv.xz") + try: + xz = lzma.LZMAFile(fname) + r1 = csv.DictReader(xz, fieldnames=["datetime", "temp-"+no, "humid-"+no]) + if frm.year == to_y and frm.month == to_m: + r1 = filter(lambda x: x["datetime"] <= to_s, r1) + elif frm.year == frm_y and frm.month == frm_m: + r1 = filter(lambda x: x["datetime"] >= frm_s , r1) + if r1 == []: continue + tmplist.append(r1) + except IOError: + break + + result += [x for s in zip(*tmplist) for x in s] + + if frm.month == 12: + frm = date(frm.year + 1, 1, 1) + else: + frm = date(frm.year, frm.month + 1, 1) # align on 10 minite for i in range(len(result)): - result[i]["datetime"] = result[i]["datetime"][:-1]+"0" - -#result.sort(key=lambda x: x["datetime"]) # maybe not needed + result[i]["datetime"] = result[i]["datetime"][:-1]+"0" + +result.sort(key=lambda x: x["datetime"]) # maybe not needed # gather all sensors prev="" fa = [] for entry in result: - if entry["datetime"] == prev: - fa[-1] = {k: v for dic in [entry, fa[-1]] - for k, v in dic.items()} - else: - fa.append(entry) - prev = entry["datetime"] + if entry["datetime"] == prev: + fa[-1] = {k: v for dic in [entry, fa[-1]] + for k, v in dic.items()} + else: + fa.append(entry) + prev = entry["datetime"] # cull data to fit about 2000 entries -print (fa) -print fa +print fa[::step]