From 0ef91e74908fb8d9dfc813c276c9c2559708f3f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asbj=C3=B8rn=20Sloth=20T=C3=B8nnesen?= Date: Mon, 4 May 2020 10:08:43 +0000 Subject: fixed to work with the new apidoc in YAML MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Asbjørn Sloth Tønnesen --- sync.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) (limited to 'sync.py') diff --git a/sync.py b/sync.py index 15d1983..cbc41ff 100755 --- a/sync.py +++ b/sync.py @@ -9,8 +9,11 @@ from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry import json - import time +import re +import yaml + +from urllib.parse import urljoin import logging from config import get_config @@ -141,6 +144,17 @@ def sync_table(s, kind): initial_sync(s, kind) fetch_objects(s, kind, { 'since': last } ) +def find_spec(s, url): + s.headers.update({'Accept': 'text/html'}) + r = s.get(url) + + # look for something like + p = re.compile(r']+\bspec-url=([\'"])([^\'">]+)\1[^>]*>') + m = p.search(r.text) + assert(m) + + return urljoin(url, m[2]) + def main(): open_db() @@ -151,14 +165,24 @@ def main(): req_agent = s.headers.get('User-Agent') s.headers.update({'User-Agent': f'peeringdb-simplesync/0.1 {req_agent:s}'}) - r = s.get('https://peeringdb.com/apidocs/') + + spec_url = find_spec(s, 'https://peeringdb.com/apidocs/') + + s.headers.update({'Accept': 'application/x-yaml'}) + r = s.get(spec_url) + + # subsequent requests are going to be JSON s.headers.update({'Accept': 'application/json'}) ignored = [ 'as_set' ] - apidoc = json.loads(r.text) - for key in apidoc['api']: - if key[0] == '_' or key in ignored: continue + apidoc = yaml.safe_load(r.text) + p = re.compile(r'^/api/([a-z_]+)$') + for path in apidoc['paths']: + m = p.match(path) + if not m: continue + key = m[1] + if key in ignored: continue try: sync_table(s, key) except AssertionError: -- cgit v1.2.1