diff options
| -rwxr-xr-x | sync.py | 12 |
1 files changed, 8 insertions, 4 deletions
@@ -92,8 +92,12 @@ def last_updated(kind): if last is None: return None return int(last) -def fetch_objects(s, kind, params): +def fetch_objects(s, kind, extra_params): endpoint = f'https://peeringdb.com/api/{kind:s}' + params = { + 'depth': 0, + **extra_params } + r = s.get(endpoint, params=params ) objs = json.loads(r.text)['data'] for obj in objs: @@ -107,7 +111,7 @@ def initial_sync(s, kind): low = 0 while lows < 5: high = low + step + 1 - n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high, 'depth': 0 } ) + n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high } ) low += step if n > 0: step = initial_step @@ -115,7 +119,7 @@ def initial_sync(s, kind): else: step *= 2 lows += 1 - fetch_objects(s, kind, { 'id__gt': low, 'depth': 0 } ) + fetch_objects(s, kind, { 'id__gt': low } ) def sync_table(s, kind): test_table(kind) @@ -124,7 +128,7 @@ def sync_table(s, kind): if last is None: last = int(time.time()) - 3600 initial_sync(s, kind) - fetch_objects(s, kind, { 'since': last, 'depth': 0 } ) + fetch_objects(s, kind, { 'since': last } ) def main(): open_db() |