fixed to work with the new apidoc in YAML

Signed-off-by: Asbjørn Sloth Tønnesen <ast@fiberby.net>

2 files changed, 31 insertions, 6 deletions

diff --git a/README.md b/README.md
index 4d613c8..6777042 100644
--- a/README.md
+++ b/README.md
@@ -10,13 +10,14 @@ Requirements
 * Python 3.6+
 * psycopg2 2.7+
 * python-requests 2+
+* python-yaml
 
 
 Installation
 -----
 
 ```sh
-sudo apt install git python3 python3-requests python3-psycopg2
+sudo apt install git python3 python3-requests python3-psycopg2 python3-yaml
 
 git clone https://git.2e8.dk/peeringdb-simplesync
 cd peeringdb-simplesync
diff --git a/sync.py b/sync.py
index 15d1983..cbc41ff 100755
--- a/sync.py
+++ b/sync.py
@@ -9,8 +9,11 @@ from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
 import json
-
 import time
+import re
+import yaml
+
+from urllib.parse import urljoin
 
 import logging
 from config import get_config
@@ -141,6 +144,17 @@ def sync_table(s, kind):
         initial_sync(s, kind)
     fetch_objects(s, kind, { 'since': last } )
 
+def find_spec(s, url):
+    s.headers.update({'Accept': 'text/html'})
+    r = s.get(url)
+
+    # look for something like <redoc spec-url='/s/2.20.2/api-schema.yaml'>
+    p = re.compile(r'<redoc[^>]+\bspec-url=([\'"])([^\'">]+)\1[^>]*>')
+    m = p.search(r.text)
+    assert(m)
+
+    return urljoin(url, m[2])
+
 def main():
     open_db()
 
@@ -151,14 +165,24 @@ def main():
 
     req_agent = s.headers.get('User-Agent')
     s.headers.update({'User-Agent': f'peeringdb-simplesync/0.1 {req_agent:s}'})
-    r = s.get('https://peeringdb.com/apidocs/')
+
+    spec_url = find_spec(s, 'https://peeringdb.com/apidocs/')
+
+    s.headers.update({'Accept': 'application/x-yaml'})
+    r = s.get(spec_url)
+
+    # subsequent requests are going to be JSON
     s.headers.update({'Accept': 'application/json'})
 
     ignored = [ 'as_set' ]
 
-    apidoc = json.loads(r.text)
-    for key in apidoc['api']:
-        if key[0] == '_' or key in ignored: continue
+    apidoc = yaml.safe_load(r.text)
+    p = re.compile(r'^/api/([a-z_]+)$')
+    for path in apidoc['paths']:
+        m = p.match(path)
+        if not m: continue
+        key = m[1]
+        if key in ignored: continue
         try:
             sync_table(s, key)
         except AssertionError: