diff options
Diffstat (limited to 'scripts/queue_runner.py')
-rw-r--r-- | scripts/queue_runner.py | 27 |
1 files changed, 14 insertions, 13 deletions
diff --git a/scripts/queue_runner.py b/scripts/queue_runner.py index 9ed52f2..fd88600 100644 --- a/scripts/queue_runner.py +++ b/scripts/queue_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -''' move items from queue to database ''' +""" move items from queue to database """ import os import pickle @@ -9,10 +9,12 @@ import time from common.postgres import dbi from litequeue import SQLQueue -QUEUE_DB = os.environ.get('el_QUEUE_db', 'litequeue.db') -QUEUE_DIR = os.environ.get('el_QUEUE_dir', 'queue') -QUEUE_DB = QUEUE_DIR + "/" + QUEUE_DB -QUEUE_SLEEP = int(os.environ.get('el_QUEUE_sleep', 15)) # Default sleep 15 seconds when queue empty +QUEUE_DB = os.environ.get("el_QUEUE_db", "litequeue.db") +QUEUE_DIR = os.environ.get("el_QUEUE_dir", "queue") +QUEUE_DB = QUEUE_DIR + "/" + QUEUE_DB +QUEUE_SLEEP = int( + os.environ.get("el_QUEUE_sleep", 15) +) # Default sleep 15 seconds when queue empty # Unlock all con = sqlite3.connect(QUEUE_DB) @@ -39,8 +41,8 @@ while True: # get message task = q.pop() - raw = pickle.loads(task['message']) - msgids.append(task['message_id']) + raw = pickle.loads(task["message"]) + msgids.append(task["message_id"]) sql = raw[0] # if the queue-item already is a batch-job, don't do any more batch-work @@ -50,28 +52,27 @@ while True: else: values.append(raw[1]) - # Check if we can batch up with the next message in queue i += 1 if i < 10 and q.qsize() - len(msgids) >= 1: - nextraw = pickle.loads(q.peek()['message']) + nextraw = pickle.loads(q.peek()["message"]) nextsql = nextraw[0] nextvalues = nextraw[1] if sql == nextsql and isinstance(nextvalues, tuple): continue - dbi(sql,values) + dbi(sql, values) for msgid in msgids: q.done(msgid) - table = sql.split(' ')[2].strip() + table = sql.split(" ")[2].strip() num = 1 if isinstance(values, tuple) else len(values) - left = str(q.qsize()) + " items left in queue" if q.qsize() > 0 else '' + left = str(q.qsize()) + " items left in queue" if q.qsize() > 0 else "" print("Processed", num, "item(s) for table", table + ".", left) msgids = [] values = [] - i=0 + i = 0 print("END") |