diff options
author | Asbjørn Sloth Tønnesen <ast@fiberby.net> | 2019-09-07 13:13:32 +0000 |
---|---|---|
committer | Asbjørn Sloth Tønnesen <ast@fiberby.net> | 2019-09-07 13:13:32 +0000 |
commit | eb381cff50bc45d3072cf005aac07c91e79dcfc6 (patch) | |
tree | 64fa13b1fa81c6db48c3c50215b52bbdb889aa68 | |
parent | 4d60befb6d323adb638941d41ba7f5fda9f070cc (diff) | |
download | peeringdb-simplesync-eb381cff50bc45d3072cf005aac07c91e79dcfc6.tar.gz peeringdb-simplesync-eb381cff50bc45d3072cf005aac07c91e79dcfc6.tar.xz peeringdb-simplesync-eb381cff50bc45d3072cf005aac07c91e79dcfc6.zip |
keep common parameters in fetch_objects()
Signed-off-by: Asbjørn Sloth Tønnesen <ast@fiberby.net>
-rwxr-xr-x | sync.py | 12 |
1 files changed, 8 insertions, 4 deletions
@@ -92,8 +92,12 @@ def last_updated(kind): if last is None: return None return int(last) -def fetch_objects(s, kind, params): +def fetch_objects(s, kind, extra_params): endpoint = f'https://peeringdb.com/api/{kind:s}' + params = { + 'depth': 0, + **extra_params } + r = s.get(endpoint, params=params ) objs = json.loads(r.text)['data'] for obj in objs: @@ -107,7 +111,7 @@ def initial_sync(s, kind): low = 0 while lows < 5: high = low + step + 1 - n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high, 'depth': 0 } ) + n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high } ) low += step if n > 0: step = initial_step @@ -115,7 +119,7 @@ def initial_sync(s, kind): else: step *= 2 lows += 1 - fetch_objects(s, kind, { 'id__gt': low, 'depth': 0 } ) + fetch_objects(s, kind, { 'id__gt': low } ) def sync_table(s, kind): test_table(kind) @@ -124,7 +128,7 @@ def sync_table(s, kind): if last is None: last = int(time.time()) - 3600 initial_sync(s, kind) - fetch_objects(s, kind, { 'since': last, 'depth': 0 } ) + fetch_objects(s, kind, { 'since': last } ) def main(): open_db() |