Adding Entsoe as a data source. Also adding Norges Bank, since Entsoes

prices are in Euros, and I need an exchange-rate.

4 files changed, 235 insertions, 0 deletions

diff --git a/entsoe2pgsql.py b/entsoe2pgsql.py
new file mode 100755
index 0000000..ef99fca
--- /dev/null
+++ b/entsoe2pgsql.py
@@ -0,0 +1,103 @@
+#!/bin/python3
+
+import os
+import sys
+import json
+import psycopg2
+import requests
+import xmltodict
+
+from datetime import datetime
+from datetime import timedelta
+from tzlocal import get_localzone
+from dateutil import tz
+
+
+# variables
+apiKey = os.environ['entsoe_token']
+apiUrl = "https://transparency.entsoe.eu/api?securityToken=" + apiKey
+
+pg_db = os.environ['pg_db']
+pg_host = os.environ['pg_host']
+pg_table = "entsoe"
+
+startTime = datetime.now(get_localzone()) - timedelta(days = 7)
+startTime = startTime.strftime('%Y%m%d')
+
+endTime = datetime.now(get_localzone()).strftime('%Y%m%d')
+
+areas = [ {"name": "NO-0", "code": "10YNO-0--------C"},
+          {"name": "NO-1", "code": "10YNO-1--------2"},
+          {"name": "NO-2", "code": "10YNO-2--------T"},
+          {"name": "NO-3", "code": "10YNO-3--------J"},
+          {"name": "NO-4", "code": "10YNO-4--------9"} ]
+    
+UTC = tz.gettz('UTC')
+CET = tz.gettz('Europe/Oslo')
+
+
+# Get the data
+values=[]
+for area in areas:
+    try:
+        url = apiUrl + "&documentType=A44&in_Domain=" + area["code"] + "&out_Domain=" + area["code"] + "&periodStart=" + startTime + "0000&periodEnd=" + endTime + "0000"
+    
+        print("Getting data for " + area["code"])
+        response = requests.get(url)
+        if response.status_code != 200:
+            print(response.status_code)
+            print("Oh shit")
+            response.raise_for_status()
+    
+    except Exception as e:
+        print("oh lol")
+        sys.exit(e)
+    
+    data_dict = xmltodict.parse(response.content)
+    
+    items = 0
+    if "Publication_MarketDocument" in data_dict:
+        for lista in data_dict["Publication_MarketDocument"]["TimeSeries"]:
+            utctime = datetime.strptime(lista["Period"]["timeInterval"]["start"], "%Y-%m-%dT%H:%MZ")
+            utctime = utctime.replace(tzinfo = UTC)
+            cettime = utctime.astimezone(CET)
+            items += len(lista["Period"]["Point"])
+
+            for item in lista["Period"]["Point"]:
+                # the response contains timerange, but not timestamp for every price, so we must calculate it
+                time = str(cettime + timedelta(hours = int(item["position"]) - 1))
+
+                # append values
+                values.append((time, area["name"], item["price.amount"]))
+    print("Got " + str(items) + " records")
+    
+
+# connect to db
+conn = psycopg2.connect(database=pg_db, host=pg_host)
+cur = conn.cursor()
+
+# count rows before we start
+cur.execute("SELECT COUNT(*) FROM " + pg_table)
+before = cur.fetchone()
+
+# insert data
+print("Inserting into database")
+try:
+    cur.executemany("INSERT INTO " + pg_table + " VALUES(%s,%s,%s) ON CONFLICT (starttime, zone) DO NOTHING", values)
+    conn.commit()
+except Exception as e:
+    conn.rollback()
+    raise e
+
+# count rows after we finnish
+cur.execute("SELECT COUNT(*) FROM " + pg_table)
+after = cur.fetchone()
+
+# count *new* rows
+newRows = after[0] - before[0]
+
+# close connection
+conn.close()
+
+
+print("Successfully inserted " + str(newRows) + " records into the database")
diff --git a/env.sample b/env.sample
index 0e1b0a5..b54a9c7 100644
--- a/env.sample
+++ b/env.sample
@@ -8,3 +8,4 @@ export tibber_token=XXX
 export yr_lat=0.0
 export yr_lon=0.0
 
+export entsoe_token=XXX
diff --git a/nb2pgsql.py b/nb2pgsql.py
new file mode 100755
index 0000000..6a6ffe3
--- /dev/null
+++ b/nb2pgsql.py
@@ -0,0 +1,84 @@
+#!/bin/python3
+
+import os
+import sys
+import csv
+import json
+import psycopg2
+import requests
+import tempfile
+
+from datetime import datetime
+from datetime import timedelta
+from tzlocal import get_localzone
+
+# I'm not sure I understand Norges Banks json-model. It seems a lot easier to just get the CSV, and convert it to JSON.
+apiUrl = "https://data.norges-bank.no/api/data/EXR/B.EUR.NOK.SP?format=csv&locale=en"
+
+pg_db = os.environ['pg_db']
+pg_host = os.environ['pg_host']
+pg_table = "nbex"
+
+startTime = datetime.now(get_localzone()) - timedelta(days = 10)
+startTime = startTime.strftime('%Y-%m-%d')
+
+endTime = datetime.now(get_localzone()).strftime('%Y-%m-%d')
+
+temp = tempfile.NamedTemporaryFile()
+
+### Get the data
+try:
+    url = apiUrl + "&startPeriod=" + startTime + "&endPeriod=" + endTime
+
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(response.status_code)
+        print("Oh shit")
+        response.raise_for_status()
+
+    with open(temp.name,'w') as fd:
+        fd.write(response.text)
+
+except Exception as e:
+    print("oh lol")
+    sys.exit(e)
+
+
+### insert data into database
+
+values = []
+with open(temp.name) as csvfile:
+    csvReader = csv.DictReader(csvfile, delimiter=';')
+
+    for item in csvReader:
+        values.append((item["TIME_PERIOD"],item["BASE_CUR"],item["QUOTE_CUR"],item["OBS_VALUE"]))
+temp.close()
+
+
+# connect to database
+conn = psycopg2.connect(database=pg_db, host=pg_host)
+cur = conn.cursor()
+
+# count rows before we start
+cur.execute("SELECT COUNT(*) FROM " + pg_table)
+before = cur.fetchone()
+
+# insert data
+try:
+    cur.executemany("INSERT INTO " + pg_table + " VALUES(%s,%s,%s,%s) ON CONFLICT (startdate,base_cur,quote_cur) DO NOTHING", values)
+    conn.commit()
+except Exception as e:
+    conn.rollback()
+    raise e
+
+# count rows after we finnish
+cur.execute("SELECT COUNT(*) FROM " + pg_table)
+after = cur.fetchone()
+
+# count *new* rows
+newRows = after[0] - before[0]
+
+# close connection
+conn.close()
+
+print("Successfully inserted " + str(newRows) + " records into the database")
diff --git a/schema.sql b/schema.sql
index 6b20b52..89d8e77 100644
--- a/schema.sql
+++ b/schema.sql
@@ -38,6 +38,37 @@ CREATE TABLE public.elvia (
 ALTER TABLE public.elvia OWNER TO dennis;
 
 --
+-- Name: entsoe; Type: TABLE; Schema: public; Owner: dennis
+--
+
+CREATE TABLE public.entsoe (
+    starttime timestamp with time zone NOT NULL,
+    zone character varying(4) NOT NULL,
+    price numeric NOT NULL,
+    currency character varying(4) DEFAULT 'EUR'::character varying NOT NULL,
+    unitmeasure character varying(4) DEFAULT 'MWH'::character varying NOT NULL,
+    inserted timestamp with time zone DEFAULT now() NOT NULL
+);
+
+
+ALTER TABLE public.entsoe OWNER TO dennis;
+
+--
+-- Name: nbex; Type: TABLE; Schema: public; Owner: dennis
+--
+
+CREATE TABLE public.nbex (
+    startdate date NOT NULL,
+    base_cur character varying(3) DEFAULT 'EUR'::character varying NOT NULL,
+    quote_cur character varying(3) DEFAULT 'NOK'::character varying NOT NULL,
+    rate numeric NOT NULL,
+    inserted timestamp with time zone DEFAULT now() NOT NULL
+);
+
+
+ALTER TABLE public.nbex OWNER TO dennis;
+
+--
 -- Name: tibber_consumption; Type: TABLE; Schema: public; Owner: dennis
 --
 
@@ -99,6 +130,22 @@ ALTER TABLE ONLY public.elvia
 
 
 --
+-- Name: entsoe entsoe_pkey; Type: CONSTRAINT; Schema: public; Owner: dennis
+--
+
+ALTER TABLE ONLY public.entsoe
+    ADD CONSTRAINT entsoe_pkey PRIMARY KEY (starttime, zone);
+
+
+--
+-- Name: nbex nbexchange_pkey; Type: CONSTRAINT; Schema: public; Owner: dennis
+--
+
+ALTER TABLE ONLY public.nbex
+    ADD CONSTRAINT nbexchange_pkey PRIMARY KEY (startdate, base_cur, quote_cur);
+
+
+--
 -- Name: tibber_consumption tibber_consumption_pkey; Type: CONSTRAINT; Schema: public; Owner: dennis
 --