From eb381cff50bc45d3072cf005aac07c91e79dcfc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asbj=C3=B8rn=20Sloth=20T=C3=B8nnesen?= Date: Sat, 7 Sep 2019 13:13:32 +0000 Subject: keep common parameters in fetch_objects() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Asbjørn Sloth Tønnesen --- sync.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sync.py b/sync.py index cf81404..5f1c3b3 100755 --- a/sync.py +++ b/sync.py @@ -92,8 +92,12 @@ def last_updated(kind): if last is None: return None return int(last) -def fetch_objects(s, kind, params): +def fetch_objects(s, kind, extra_params): endpoint = f'https://peeringdb.com/api/{kind:s}' + params = { + 'depth': 0, + **extra_params } + r = s.get(endpoint, params=params ) objs = json.loads(r.text)['data'] for obj in objs: @@ -107,7 +111,7 @@ def initial_sync(s, kind): low = 0 while lows < 5: high = low + step + 1 - n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high, 'depth': 0 } ) + n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high } ) low += step if n > 0: step = initial_step @@ -115,7 +119,7 @@ def initial_sync(s, kind): else: step *= 2 lows += 1 - fetch_objects(s, kind, { 'id__gt': low, 'depth': 0 } ) + fetch_objects(s, kind, { 'id__gt': low } ) def sync_table(s, kind): test_table(kind) @@ -124,7 +128,7 @@ def sync_table(s, kind): if last is None: last = int(time.time()) - 3600 initial_sync(s, kind) - fetch_objects(s, kind, { 'since': last, 'depth': 0 } ) + fetch_objects(s, kind, { 'since': last } ) def main(): open_db() -- cgit v1.2.1