aboutsummaryrefslogtreecommitdiffstats
path: root/scripts/queue_runner.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/queue_runner.py')
-rw-r--r--scripts/queue_runner.py27
1 files changed, 14 insertions, 13 deletions
diff --git a/scripts/queue_runner.py b/scripts/queue_runner.py
index 9ed52f2..fd88600 100644
--- a/scripts/queue_runner.py
+++ b/scripts/queue_runner.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-''' move items from queue to database '''
+""" move items from queue to database """
import os
import pickle
@@ -9,10 +9,12 @@ import time
from common.postgres import dbi
from litequeue import SQLQueue
-QUEUE_DB = os.environ.get('el_QUEUE_db', 'litequeue.db')
-QUEUE_DIR = os.environ.get('el_QUEUE_dir', 'queue')
-QUEUE_DB = QUEUE_DIR + "/" + QUEUE_DB
-QUEUE_SLEEP = int(os.environ.get('el_QUEUE_sleep', 15)) # Default sleep 15 seconds when queue empty
+QUEUE_DB = os.environ.get("el_QUEUE_db", "litequeue.db")
+QUEUE_DIR = os.environ.get("el_QUEUE_dir", "queue")
+QUEUE_DB = QUEUE_DIR + "/" + QUEUE_DB
+QUEUE_SLEEP = int(
+ os.environ.get("el_QUEUE_sleep", 15)
+) # Default sleep 15 seconds when queue empty
# Unlock all
con = sqlite3.connect(QUEUE_DB)
@@ -39,8 +41,8 @@ while True:
# get message
task = q.pop()
- raw = pickle.loads(task['message'])
- msgids.append(task['message_id'])
+ raw = pickle.loads(task["message"])
+ msgids.append(task["message_id"])
sql = raw[0]
# if the queue-item already is a batch-job, don't do any more batch-work
@@ -50,28 +52,27 @@ while True:
else:
values.append(raw[1])
-
# Check if we can batch up with the next message in queue
i += 1
if i < 10 and q.qsize() - len(msgids) >= 1:
- nextraw = pickle.loads(q.peek()['message'])
+ nextraw = pickle.loads(q.peek()["message"])
nextsql = nextraw[0]
nextvalues = nextraw[1]
if sql == nextsql and isinstance(nextvalues, tuple):
continue
- dbi(sql,values)
+ dbi(sql, values)
for msgid in msgids:
q.done(msgid)
- table = sql.split(' ')[2].strip()
+ table = sql.split(" ")[2].strip()
num = 1 if isinstance(values, tuple) else len(values)
- left = str(q.qsize()) + " items left in queue" if q.qsize() > 0 else ''
+ left = str(q.qsize()) + " items left in queue" if q.qsize() > 0 else ""
print("Processed", num, "item(s) for table", table + ".", left)
msgids = []
values = []
- i=0
+ i = 0
print("END")