4 files changed, 106 insertions, 33 deletions
diff --git a/README.md b/README.md
index 6777042..7de1e8b 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ git clone https://git.2e8.dk/peeringdb-simplesync
 cd peeringdb-simplesync
 
 cp config.py.sample config.py
-editor config.py # change credentials
+editor config.py # provide API-key
 
 createdb pdbtest
 psql pdbtest < schema.sql
diff --git a/config.py.sample b/config.py.sample
index 42bbac4..9e3ba44 100644
--- a/config.py.sample
+++ b/config.py.sample
@@ -1,21 +1,38 @@
-from requests.auth import HTTPBasicAuth
 #
 # Config keys
 # ===========
 #
+# domain_name: PeeringDB domain name
+#   Defaults to "www.peeringdb.com",
+#   but can eg. be set to "beta.peeringdb.com" for testing
+#
+# object_types: array of object types to sync
+#   defaults to finding them through the machine-readable
+#   API documentaion, but if that is too slow (parsing a big YAML file),
+#   then you can just set it statically.
+#   This is also useful if you only want a subset of object types:
+#   Example: 'object_types': [ 'org', 'net', 'poc' ]
+#
 # db_conn_str: libpq database connection string
 #   https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
 #
 # db_schema: database schema name
 #   If not specified, it defaults to 'peeringdb'
 #
+# auth: 'API-key'
+#   Authenticate using an API-key.
+#
+#   https://docs.peeringdb.com/blog/api_keys/
+#   https://github.com/peeringdb/peeringdb/blob/master/docs/api_keys.md
+#
 # auth: HTTPBasicAuth('username', 'password')
-#   Authenticate using username/password
+#   Authenticate using username/password (DEPRECATED)
+#   requires: from requests.auth import HTTPBasicAuth
 #
 
 def get_config():
     return {
         'db_conn_str': 'dbname=pdbtest',
         'db_schema': 'peeringdb',
-        'auth': HTTPBasicAuth('use_a_dedicated_peeringdb_account', 'and_a_long_random_password'),
+        'auth': 'obtain_an_api-key_from_peeringdb_com',
     }
diff --git a/schema.sql b/schema.sql
index f658f38..6b4d30f 100644
--- a/schema.sql
+++ b/schema.sql
@@ -1,6 +1,6 @@
-create schema peeringdb;
+CREATE SCHEMA IF NOT EXISTS peeringdb;
 
-create table peeringdb.org (
+CREATE TABLE IF NOT EXISTS peeringdb.org (
 	id int not null,
 	status text not null,
 	data jsonb not null,
@@ -10,7 +10,7 @@ create table peeringdb.org (
 	primary key (id)
 );
 
-create table peeringdb.net (
+CREATE TABLE IF NOT EXISTS peeringdb.net (
 	id int not null,
 	org_id int not null,
 	asn bigint not null,
@@ -22,7 +22,7 @@ create table peeringdb.net (
 	primary key (id)
 );
 
-create table peeringdb.ix (
+CREATE TABLE IF NOT EXISTS peeringdb.ix (
 	id int not null,
 	org_id int not null,
 	status text not null,
@@ -33,9 +33,10 @@ create table peeringdb.ix (
 	primary key (id)
 );
 
-create table peeringdb.fac (
+CREATE TABLE IF NOT EXISTS peeringdb.fac (
 	id int not null,
 	org_id int not null,
+	campus_id int,
 	status text not null,
 	data jsonb not null,
 	created timestamptz not null,
@@ -44,7 +45,7 @@ create table peeringdb.fac (
 	primary key (id)
 );
 
-create table peeringdb.poc (
+CREATE TABLE IF NOT EXISTS peeringdb.poc (
 	id int not null,
 	net_id int not null,
 	status text not null,
@@ -55,7 +56,7 @@ create table peeringdb.poc (
 	primary key (id)
 );
 
-create table peeringdb.ixlan (
+CREATE TABLE IF NOT EXISTS peeringdb.ixlan (
 	id int not null,
 	ix_id int not null,
 	status text not null,
@@ -66,7 +67,7 @@ create table peeringdb.ixlan (
 	primary key (id)
 );
 
-create table peeringdb.ixpfx (
+CREATE TABLE IF NOT EXISTS peeringdb.ixpfx (
 	id int not null,
 	ixlan_id int not null,
 	status text not null,
@@ -77,7 +78,7 @@ create table peeringdb.ixpfx (
 	primary key (id)
 );
 
-create table peeringdb.ixfac (
+CREATE TABLE IF NOT EXISTS peeringdb.ixfac (
 	id int not null,
 	ix_id int not null,
 	fac_id int not null,
@@ -89,7 +90,7 @@ create table peeringdb.ixfac (
 	primary key (id)
 );
 
-create table peeringdb.netfac (
+CREATE TABLE IF NOT EXISTS peeringdb.netfac (
 	id int not null,
 	net_id int not null,
 	fac_id int not null,
@@ -101,7 +102,7 @@ create table peeringdb.netfac (
 	primary key (id)
 );
 
-create table peeringdb.netixlan (
+CREATE TABLE IF NOT EXISTS peeringdb.netixlan (
 	id int not null,
 	net_id int not null,
 	ix_id int not null,
@@ -113,3 +114,37 @@ create table peeringdb.netixlan (
 	deleted timestamptz,
 	primary key (id)
 );
+
+CREATE TABLE IF NOT EXISTS peeringdb.carrier (
+	id int not null,
+	org_id int not null,
+	status text not null,
+	data jsonb not null,
+	created timestamptz not null,
+	updated timestamptz not null,
+	deleted timestamptz,
+	primary key (id)
+);
+
+CREATE TABLE IF NOT EXISTS peeringdb.carrierfac (
+	id int not null,
+	carrier_id int not null,
+	fac_id int not null,
+	status text not null,
+	data jsonb not null,
+	created timestamptz not null,
+	updated timestamptz not null,
+	deleted timestamptz,
+	primary key (id)
+);
+
+CREATE TABLE IF NOT EXISTS peeringdb.campus (
+	id int not null,
+	org_id int not null,
+	status text not null,
+	data jsonb not null,
+	created timestamptz not null,
+	updated timestamptz not null,
+	deleted timestamptz,
+	primary key (id)
+);
diff --git a/sync.py b/sync.py
index bd26211..c9e8820 100755
--- a/sync.py
+++ b/sync.py
@@ -19,6 +19,8 @@ import logging
 from config import get_config
 
 schema_name = get_config().get('db_schemad', 'peeringdb')
+domain_name = get_config().get('domain_name', 'www.peeringdb.com')
+user_agent = 'peeringdb-simplesync/0.2'
 
 #logging.basicConfig(level=logging.DEBUG)
 
@@ -106,7 +108,7 @@ def last_updated(kind):
     return int(last)
 
 def fetch_objects(s, kind, extra_params):
-    endpoint = f'https://peeringdb.com/api/{kind:s}'
+    endpoint = f'https://{domain_name:s}/api/{kind:s}'
     params = {
             'depth': 0,
             'status__in': 'ok,pending,deleted',
@@ -138,7 +140,7 @@ def initial_sync(s, kind):
 
 def sync_table(s, kind):
     test_table(kind)
-    endpoint = f'https://peeringdb.com/api/{kind:s}'
+    endpoint = f'https://{domain_name:s}/api/{kind:s}'
     last = last_updated(kind)
     if last is None:
         last = int(time.time()) - 3600
@@ -156,34 +158,53 @@ def find_spec(s, url):
 
     return urljoin(url, m[2])
 
+def find_object_types_via_apidocs(s):
+    ret = []
+    spec_url = find_spec(s, f'https://{domain_name:s}/apidocs/')
+    s.headers.update({'Accept': 'application/x-yaml'})
+    r = s.get(spec_url)
+    ignored_types = [ 'as_set' ]
+    apidoc = yaml.safe_load(r.text)
+    p = re.compile(r'^/api/([a-z_]+)$')
+    for path in apidoc['paths']:
+        m = p.match(path)
+        if not m: continue
+        key = m[1]
+        if key in ignored_types: continue
+        ret.append(key)
+    return ret
+
+def handle_auth(s):
+    auth = get_config().get('auth')
+    if type(auth) == str:
+        # API-Key
+        s.headers.update({'Authorization': f'Api-Key {auth:s}'})
+    else:
+        # eg. HTTPBasicAuth('username', 'password')
+        s.auth = auth
+
 def main():
     open_db()
 
     s = requests.Session()
-    retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 502, 503, 504 ])
+    retries = Retry(total=16,
+                    backoff_factor=1.5,
+                    status_forcelist=[ 429, 502, 503, 504 ])
     s.mount('https://', HTTPAdapter(max_retries=retries))
-    s.auth = get_config()['auth']
+    handle_auth(s)
 
     req_agent = s.headers.get('User-Agent')
-    s.headers.update({'User-Agent': f'peeringdb-simplesync/0.1 {req_agent:s}'})
+    s.headers.update({'User-Agent': f'{user_agent:s} {req_agent:s}'})
 
-    spec_url = find_spec(s, 'https://peeringdb.com/apidocs/')
-
-    s.headers.update({'Accept': 'application/x-yaml'})
-    r = s.get(spec_url)
+    keys = get_config().get('object_types')
+    if not keys:
+        keys = find_object_types_via_apidocs(s)
+        print('Consider setting \'object_types\':', keys)
 
     # subsequent requests are going to be JSON
     s.headers.update({'Accept': 'application/json'})
 
-    ignored = [ 'as_set' ]
-
-    apidoc = yaml.safe_load(r.text)
-    p = re.compile(r'^/api/([a-z_]+)$')
-    for path in apidoc['paths']:
-        m = p.match(path)
-        if not m: continue
-        key = m[1]
-        if key in ignored: continue
+    for key in keys:
         try:
             sync_table(s, key)
         except AssertionError: