diff options
Diffstat (limited to 'entsoe2pgsql.py')
| -rwxr-xr-x | entsoe2pgsql.py | 52 |
1 files changed, 11 insertions, 41 deletions
diff --git a/entsoe2pgsql.py b/entsoe2pgsql.py index 3fdb6b6..2597a98 100755 --- a/entsoe2pgsql.py +++ b/entsoe2pgsql.py @@ -2,21 +2,20 @@ import os import sys -import json -import psycopg2 -import requests -import xmltodict - from datetime import datetime from datetime import timedelta +import requests +import xmltodict from tzlocal import get_localzone from dateutil import tz +from common import dbi + # variables # Getting an api-key isn't very well documented. The documentation [1] points -# to a pdf [2], which says the following: +# to a pdf [2], which says the following: # > In order to request the access to the Restful API, please register on the # > Transparency Platform and send an email to transparency@entsoe.eu with # > “Restful API access” in the subject line. Indicate the email address you @@ -45,7 +44,7 @@ areas = [ {"name": "NO-0", "code": "10YNO-0--------C"}, {"name": "NO-2", "code": "10YNO-2--------T"}, {"name": "NO-3", "code": "10YNO-3--------J"}, {"name": "NO-4", "code": "10YNO-4--------9"} ] - + UTC = tz.gettz('UTC') CET = tz.gettz('Europe/Oslo') @@ -55,20 +54,20 @@ values=[] for area in areas: try: url = apiUrl + "&documentType=A44&in_Domain=" + area["code"] + "&out_Domain=" + area["code"] + "&periodStart=" + startTime + "0000&periodEnd=" + endTime + "0000" - + print("Getting data for " + area["code"]) response = requests.get(url) if response.status_code != 200: print(response.status_code) print("Oh shit") response.raise_for_status() - + except Exception as e: print("oh lol") sys.exit(e) - + data_dict = xmltodict.parse(response.content) - + items = 0 if "Publication_MarketDocument" in data_dict: for lista in data_dict["Publication_MarketDocument"]["TimeSeries"]: @@ -84,34 +83,5 @@ for area in areas: # append values values.append((time, area["name"], item["price.amount"])) print("Got " + str(items) + " records") - - -# connect to db -conn = psycopg2.connect(database=pg_db, host=pg_host) -cur = conn.cursor() - -# count rows before we start -cur.execute("SELECT COUNT(*) FROM " + pg_table) -before = cur.fetchone() - -# insert data -print("Inserting into database") -try: - cur.executemany("INSERT INTO " + pg_table + " VALUES(%s,%s,%s) ON CONFLICT (starttime, zone) DO NOTHING", values) - conn.commit() -except Exception as e: - conn.rollback() - raise e - -# count rows after we finnish -cur.execute("SELECT COUNT(*) FROM " + pg_table) -after = cur.fetchone() - -# count *new* rows -newRows = after[0] - before[0] - -# close connection -conn.close() - -print("Successfully inserted " + str(newRows) + " records into the database") +dbi("INSERT INTO " + pg_table + " VALUES(%s,%s,%s) ON CONFLICT (starttime, zone) DO NOTHING", values, verbose=True) |