diff --git a/.gitignore b/.gitignore index 3c9b48b..7bf35c8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.csv *.json *.swp +.python-version +data/ diff --git a/merge.py b/merge.py index ea8a1b0..a2ba76b 100755 --- a/merge.py +++ b/merge.py @@ -1,30 +1,30 @@ -#!/usr/bin/python +#!/usr/bin/env python import csv -import lzma +from backports import lzma from datetime import date, timedelta result = [] -sensorlist = [1,2,3,4,5,6,7,8,9,11,12,13,14,15,16,17,18,19,20] +sensorlist = [1,2,4,8,11,12,13,14,15,16,17,18,19,20] +#sensorlist = [8] def gather_data(frm, to, which): - global result - if which == 0: - dbdate = frm - else: - dbdate = to - dname = "%04d%02d" % (dbdate.year, dbdate.month) - frm_s = frm.strftime("%Y/%m/%d 00:00") - to_s = to.strftime("%Y/%m/%d 00:00") - tmplist = [] - for i in sensorlist: - no = str(i) - fname = "/home/tutimura/public_html/cgi-bin/ondotori/data/"+no+"/"+dname+".csv.xz" - r1 = csv.DictReader(lzma.LZMAFile(fname), fieldnames=["datetime", "temp-"+no, "humid-"+no]) - tmp=filter(lambda x: x["datetime"] >= frm_s and x["datetime"] <= to_s, r1) - if tmp == []: continue - tmplist.append(tmp) - result += [x for s in zip(*tmplist) for x in s] + global result + if which == 0: + dbdate = frm + else: + dbdate = to + dname = "%04d%02d" % (dbdate.year, dbdate.month) + frm_s = frm.strftime("%Y/%m/%d 00:00") + to_s = to.strftime("%Y/%m/%d 23:59") + tmplist = [] + for i in sensorlist: + no = str(i) + fname = "./data/"+no+"/"+dname+".csv.xz" + r1 = csv.DictReader(lzma.LZMAFile(fname), fieldnames=["datetime", "temp-"+no, "humid-"+no]) + tmp=filter(lambda x: x["datetime"] >= frm_s and x["datetime"] <= to_s, r1) + if tmp == []: continue + result+=(tmp) to = date.today() frm = to - timedelta(15) @@ -34,6 +34,7 @@ gather_data(frm, to, 1) # align on 10 min. + for i in range(len(result)): result[i]["datetime"] = result[i]["datetime"][:-1]+"0" diff --git a/update.py b/update.py index f18f48a..9b7e5c4 100755 --- a/update.py +++ b/update.py @@ -1,74 +1,75 @@ -#!/usr/bin/python +#!/usr/bin/env python import csv -import lzma +from backports import lzma import sys import os from datetime import date n_returns = 600 -sensorlist = [1,2,3,4,5,6,7,8,9,11,12,13,14,15,16,17,18,19,20] +sensorlist = [1,2,4,8,11,12,13,14,15,16,17,18,19,20] + +frm_s = sys.argv[1] +to_s = sys.argv[2] +#div = sys.argv[3] + +frm_y, frm_m, frm_d = [int(x) for x in frm_s.split('/')] +to_y, to_m, to_d = [int(x) for x in to_s.split('/')] -frm = sys.argv[1] -to = sys.argv[2] -div = sys.argv[3] - -frm_y, frm_m, frm_d = [int(x) for x in frm.split('/')] -to_y, to_m, to_d = [int(x) for x in to.split('/')] - -frm = date(frm_y, frm_m, frm_d) # align for naughty input -to = date(to_y, to_m, to_d) # align for naughty input - -# retreive from database -# first, calculate culling idx +# first, calculate culling step +frm = date(frm_y, frm_m, frm_d) +to = date(to_y, to_m, to_d) +frm_s = frm.strftime("%Y/%m/%d 00:00") # refine input +to_s = to.strftime("%Y/%m/%d 23:59") # refine input samples_per_day = 24 * 6 ndays = (to - frm).days * samples_per_day step = int(ndays / n_returns) + 1 +# retreive from database result = [] while frm.year < to.year or frm.month <= to.month: - tmplist = [] - for i in sensorlist: - no = str(i) - fname = "/home/tutimura/public_html/cgi-bin/ondotori/data/"+ no+ - "/"+frm.strftime("%Y%m")+".csv.xz" - try: - xz = lzma.LZMAFile(fname) - r1 = csv.DictReader(xz, fieldnames=["datetime", "temp-"+no, "humid-"+no]) - frm_s = frm.strftime("%Y/%m/%d 00:00") - to_s = to.strftime("%Y/%m/%d 00:00") - tmp=filter(lambda x: x["datetime"] >= frm.strftime("%Y/%m/%d") + " 00:00" - and x["datetime"] <= to + " 23:59", r1) - if tmp == []: continue - tmplist.append(tmp[::step]) - except IOError: - break - result += [x for s in zip(*tmplist) for x in s] + tmplist = [] - frm_m += 1 - if frm_m > 12: - frm_y += 1 - frm_m = 1 + for i in sensorlist: + no = str(i) + fname = "data/"+ no+ frm.strftime("/%Y%m.csv.xz") + try: + xz = lzma.LZMAFile(fname) + r1 = csv.DictReader(xz, fieldnames=["datetime", "temp-"+no, "humid-"+no]) + if frm.year == to_y and frm.month == to_m: + r1 = filter(lambda x: x["datetime"] <= to_s, r1) + elif frm.year == frm_y and frm.month == frm_m: + r1 = filter(lambda x: x["datetime"] >= frm_s , r1) + if r1 == []: continue + tmplist.append(r1) + except IOError: + break + + result += [x for s in zip(*tmplist) for x in s] + + if frm.month == 12: + frm = date(frm.year + 1, 1, 1) + else: + frm = date(frm.year, frm.month + 1, 1) # align on 10 minite for i in range(len(result)): - result[i]["datetime"] = result[i]["datetime"][:-1]+"0" - -#result.sort(key=lambda x: x["datetime"]) # maybe not needed + result[i]["datetime"] = result[i]["datetime"][:-1]+"0" + +result.sort(key=lambda x: x["datetime"]) # maybe not needed # gather all sensors prev="" fa = [] for entry in result: - if entry["datetime"] == prev: - fa[-1] = {k: v for dic in [entry, fa[-1]] - for k, v in dic.items()} - else: - fa.append(entry) - prev = entry["datetime"] + if entry["datetime"] == prev: + fa[-1] = {k: v for dic in [entry, fa[-1]] + for k, v in dic.items()} + else: + fa.append(entry) + prev = entry["datetime"] # cull data to fit about 2000 entries -print (fa) -print fa +print fa[::step]