diff --git a/merge.py b/merge.py index bf0e5ac..ea8a1b0 100755 --- a/merge.py +++ b/merge.py @@ -2,20 +2,38 @@ import csv import lzma +from datetime import date, timedelta result = [] -sensorlist = [1,2,4,8,11,12,13,14,15,16,17,18,19,20] -for i in sensorlist: - no = str(i) - r1 = csv.DictReader(lzma.LZMAFile("/home/tutimura/public_html/cgi-bin/ondotori/data/"+no+"/201612.csv.xz"), fieldnames=["datetime", "temp-"+no, "humid-"+no]) - result+=list(r1) +sensorlist = [1,2,3,4,5,6,7,8,9,11,12,13,14,15,16,17,18,19,20] -for i in sensorlist: - no = str(i) - r1 = csv.DictReader(lzma.LZMAFile("/home/tutimura/public_html/cgi-bin/ondotori/data/"+no+"/201701.csv.xz"), fieldnames=["datetime", "temp-"+no, "humid-"+no]) - result+=list(r1) +def gather_data(frm, to, which): + global result + if which == 0: + dbdate = frm + else: + dbdate = to + dname = "%04d%02d" % (dbdate.year, dbdate.month) + frm_s = frm.strftime("%Y/%m/%d 00:00") + to_s = to.strftime("%Y/%m/%d 00:00") + tmplist = [] + for i in sensorlist: + no = str(i) + fname = "/home/tutimura/public_html/cgi-bin/ondotori/data/"+no+"/"+dname+".csv.xz" + r1 = csv.DictReader(lzma.LZMAFile(fname), fieldnames=["datetime", "temp-"+no, "humid-"+no]) + tmp=filter(lambda x: x["datetime"] >= frm_s and x["datetime"] <= to_s, r1) + if tmp == []: continue + tmplist.append(tmp) + result += [x for s in zip(*tmplist) for x in s] -#result = sum(result,[]) #flatten list of list +to = date.today() +frm = to - timedelta(15) + +gather_data(frm, to, 0) +if frm.month != to.month: + gather_data(frm, to, 1) + +# align on 10 min. for i in range(len(result)): result[i]["datetime"] = result[i]["datetime"][:-1]+"0" @@ -31,7 +49,5 @@ fa.append(entry) prev = entry["datetime"] - -fa = fa[::2] print "var data = " print fa diff --git a/update.py b/update.py new file mode 100755 index 0000000..f18f48a --- /dev/null +++ b/update.py @@ -0,0 +1,74 @@ +#!/usr/bin/python + +import csv +import lzma +import sys +import os +from datetime import date + +n_returns = 600 +sensorlist = [1,2,3,4,5,6,7,8,9,11,12,13,14,15,16,17,18,19,20] + + +frm = sys.argv[1] +to = sys.argv[2] +div = sys.argv[3] + +frm_y, frm_m, frm_d = [int(x) for x in frm.split('/')] +to_y, to_m, to_d = [int(x) for x in to.split('/')] + +frm = date(frm_y, frm_m, frm_d) # align for naughty input +to = date(to_y, to_m, to_d) # align for naughty input + +# retreive from database +# first, calculate culling idx +samples_per_day = 24 * 6 +ndays = (to - frm).days * samples_per_day +step = int(ndays / n_returns) + 1 + +result = [] +while frm.year < to.year or frm.month <= to.month: + tmplist = [] + for i in sensorlist: + no = str(i) + fname = "/home/tutimura/public_html/cgi-bin/ondotori/data/"+ no+ + "/"+frm.strftime("%Y%m")+".csv.xz" + try: + xz = lzma.LZMAFile(fname) + r1 = csv.DictReader(xz, fieldnames=["datetime", "temp-"+no, "humid-"+no]) + frm_s = frm.strftime("%Y/%m/%d 00:00") + to_s = to.strftime("%Y/%m/%d 00:00") + tmp=filter(lambda x: x["datetime"] >= frm.strftime("%Y/%m/%d") + " 00:00" + and x["datetime"] <= to + " 23:59", r1) + if tmp == []: continue + tmplist.append(tmp[::step]) + except IOError: + break + result += [x for s in zip(*tmplist) for x in s] + + frm_m += 1 + if frm_m > 12: + frm_y += 1 + frm_m = 1 + +# align on 10 minite +for i in range(len(result)): + result[i]["datetime"] = result[i]["datetime"][:-1]+"0" + +#result.sort(key=lambda x: x["datetime"]) # maybe not needed + +# gather all sensors +prev="" +fa = [] +for entry in result: + if entry["datetime"] == prev: + fa[-1] = {k: v for dic in [entry, fa[-1]] + for k, v in dic.items()} + else: + fa.append(entry) + prev = entry["datetime"] + +# cull data to fit about 2000 entries +print (fa) +print fa +