Change to new API

* Should speed things up, because batch mode is supported

* API has proper information about rate limiting to we could work around
  it

* API could be configured to just deliver what we need (saves bandwidth)

main.py

@@ -72,24 +72,38 @@ def checkActiveOptions(page,items):
 
     return False;
 
-
-def getCountry(domain):
-    with urllib.request.urlopen("https://tools.keycdn.com/geo.json?host=" + domain) as url:
-            data = json.loads(url.read().decode())
-            if data['status'] == "success":
-                return data['data']['geo']['country_code']
-            else:
-                return "n.A."
-
-def checkHoster(domain):
-    with urllib.request.urlopen("https://tools.keycdn.com/geo.json?host=" + domain) as url:
-            data = json.loads(url.read().decode())
-            if data['status'] == "success":
-                for asnDesc in badAsnDesc:
-                    if data['data']['geo']['isp'].casefold().find(asnDesc.casefold()) != -1:
-                        return True, data['data']['geo']['isp']
-                return False, data['data']['geo']['isp']
-            return False, "n.A."
+def checkHoster(result):
+    if result['status'] == "success":
+        for asnDesc in badAsnDesc:
+            if result['as'].casefold().find(asnDesc.casefold()) != -1:
+                return True
+    return False
+
+def getDataFromAPI(domains):
+    results = {}
+    current = []
+    while domains != []:
+        if len(domains) >= 100:
+            current = domains[:100]
+            domains = domains[100:]
+        else:
+            current = domains
+            domains = []
+        ips = [None] * len(current)
+        for i in range(len(current)):
+            try:
+                ips[i] = socket.gethostbyname(current[i])
+            except socket.gaierror as err:
+                sys.stderr.write(str(err) + "\n")
+        r = httpx.post('http://ip-api.com/batch?fields=status,countryCode,isp,as,query', json=ips)
+        answers = r.json()
+        for i in range(len(current)):
+            results[current[i]] = answers[i]
+        if r.headers['X-Rl'] == 0:
+            time.sleep(r.headers['X-Ttl'])
+        current = []
+        ips = []
+    return results
 
 def parseText(f):
     char = f.read(1)
@@ -163,28 +177,36 @@ def main():
     i = 0
     outputs = []
     notAccessible = []
-    for domain in domains:
-        sys.stderr.write(f"{i}/{total} done, ")
-        i = i + 1
-        sys.stderr.write("doing " + domain + " now\n")
+    sys.stderr.write("Doing API request now\n")
+    apiResults = getDataFromAPI(domains)
+    for domain, result in apiResults.items():
+        sys.stderr.write("Doing page " + domain + " now\n")
 
         page = getPage(domain)
 
         available = True if page else False
 
-        corporateHoster, hoster = checkHoster(domain)
-
-        output = {
-            "domain": domain,
-            "available": available,
-            "corporateHoster": corporateHoster,
-            "hoster": hoster,
-            "countryCode": getCountry(domain),
-            "corporateStun": checkActiveOptions(page,bad_stuns),
-            "jitsiVersion": getJitsiVersion(page),
-            "authEnabled": checkActiveOptions(page,authOptions),
-            "analyticsEnabled": checkActiveOptions(page,analyticsOptions),
-        }
+        output = {}
+        if available == True and result['status'] == "success":
+            output = {
+                "domain": domain,
+                "available": True,
+                "corporateHoster": checkHoster(result),
+                "hoster": result['isp'],
+                "countryCode": result['countryCode'],
+                "corporateStun": checkActiveOptions(page,bad_stuns),
+                "jitsiVersion": getJitsiVersion(page),
+                "authEnabled": checkActiveOptions(page,authOptions),
+                "analyticsEnabled": checkActiveOptions(page,analyticsOptions),
+            }
+        else:
+            output = {
+                "domain": domain,
+                "available": False,
+                "countryCode": "",
+            }
+            if result['status'] == "success":
+                output['countryCode'] = result['countryCode']
 
         if available:
             outputs.append(output)