aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAsbjørn Sloth Tønnesen <ast@fiberby.net>2019-09-07 13:13:32 +0000
committerAsbjørn Sloth Tønnesen <ast@fiberby.net>2019-09-07 13:13:32 +0000
commiteb381cff50bc45d3072cf005aac07c91e79dcfc6 (patch)
tree64fa13b1fa81c6db48c3c50215b52bbdb889aa68
parent4d60befb6d323adb638941d41ba7f5fda9f070cc (diff)
downloadpeeringdb-simplesync-eb381cff50bc45d3072cf005aac07c91e79dcfc6.tar.gz
peeringdb-simplesync-eb381cff50bc45d3072cf005aac07c91e79dcfc6.tar.xz
peeringdb-simplesync-eb381cff50bc45d3072cf005aac07c91e79dcfc6.zip
keep common parameters in fetch_objects()
Signed-off-by: Asbjørn Sloth Tønnesen <ast@fiberby.net>
-rwxr-xr-xsync.py12
1 files changed, 8 insertions, 4 deletions
diff --git a/sync.py b/sync.py
index cf81404..5f1c3b3 100755
--- a/sync.py
+++ b/sync.py
@@ -92,8 +92,12 @@ def last_updated(kind):
if last is None: return None
return int(last)
-def fetch_objects(s, kind, params):
+def fetch_objects(s, kind, extra_params):
endpoint = f'https://peeringdb.com/api/{kind:s}'
+ params = {
+ 'depth': 0,
+ **extra_params }
+
r = s.get(endpoint, params=params )
objs = json.loads(r.text)['data']
for obj in objs:
@@ -107,7 +111,7 @@ def initial_sync(s, kind):
low = 0
while lows < 5:
high = low + step + 1
- n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high, 'depth': 0 } )
+ n = fetch_objects(s, kind, { 'id__gt': low, 'id__lt': high } )
low += step
if n > 0:
step = initial_step
@@ -115,7 +119,7 @@ def initial_sync(s, kind):
else:
step *= 2
lows += 1
- fetch_objects(s, kind, { 'id__gt': low, 'depth': 0 } )
+ fetch_objects(s, kind, { 'id__gt': low } )
def sync_table(s, kind):
test_table(kind)
@@ -124,7 +128,7 @@ def sync_table(s, kind):
if last is None:
last = int(time.time()) - 3600
initial_sync(s, kind)
- fetch_objects(s, kind, { 'since': last, 'depth': 0 } )
+ fetch_objects(s, kind, { 'since': last } )
def main():
open_db()