Browse Source

Change to new API

* Should speed things up, because batch mode is supported

* API has proper information about rate limiting to we could work around
  it

* API could be configured to just deliver what we need (saves bandwidth)
tags/v0.2^0
Julian 4 months ago
parent
commit
098b61c2ef
Signed by: j.r GPG Key ID: E74A18B933F9145E
1 changed files with 57 additions and 35 deletions
  1. +57
    -35
      main.py

+ 57
- 35
main.py View File

@@ -72,24 +72,38 @@ def checkActiveOptions(page,items):

return False;


def getCountry(domain):
with urllib.request.urlopen("https://tools.keycdn.com/geo.json?host=" + domain) as url:
data = json.loads(url.read().decode())
if data['status'] == "success":
return data['data']['geo']['country_code']
else:
return "n.A."

def checkHoster(domain):
with urllib.request.urlopen("https://tools.keycdn.com/geo.json?host=" + domain) as url:
data = json.loads(url.read().decode())
if data['status'] == "success":
for asnDesc in badAsnDesc:
if data['data']['geo']['isp'].casefold().find(asnDesc.casefold()) != -1:
return True, data['data']['geo']['isp']
return False, data['data']['geo']['isp']
return False, "n.A."
def checkHoster(result):
if result['status'] == "success":
for asnDesc in badAsnDesc:
if result['as'].casefold().find(asnDesc.casefold()) != -1:
return True
return False

def getDataFromAPI(domains):
results = {}
current = []
while domains != []:
if len(domains) >= 100:
current = domains[:100]
domains = domains[100:]
else:
current = domains
domains = []
ips = [None] * len(current)
for i in range(len(current)):
try:
ips[i] = socket.gethostbyname(current[i])
except socket.gaierror as err:
sys.stderr.write(str(err) + "\n")
r = httpx.post('http://ip-api.com/batch?fields=status,countryCode,isp,as,query', json=ips)
answers = r.json()
for i in range(len(current)):
results[current[i]] = answers[i]
if r.headers['X-Rl'] == 0:
time.sleep(r.headers['X-Ttl'])
current = []
ips = []
return results

def parseText(f):
char = f.read(1)
@@ -163,28 +177,36 @@ def main():
i = 0
outputs = []
notAccessible = []
for domain in domains:
sys.stderr.write(f"{i}/{total} done, ")
i = i + 1
sys.stderr.write("doing " + domain + " now\n")
sys.stderr.write("Doing API request now\n")
apiResults = getDataFromAPI(domains)
for domain, result in apiResults.items():
sys.stderr.write("Doing page " + domain + " now\n")

page = getPage(domain)

available = True if page else False

corporateHoster, hoster = checkHoster(domain)

output = {
"domain": domain,
"available": available,
"corporateHoster": corporateHoster,
"hoster": hoster,
"countryCode": getCountry(domain),
"corporateStun": checkActiveOptions(page,bad_stuns),
"jitsiVersion": getJitsiVersion(page),
"authEnabled": checkActiveOptions(page,authOptions),
"analyticsEnabled": checkActiveOptions(page,analyticsOptions),
}
output = {}
if available == True and result['status'] == "success":
output = {
"domain": domain,
"available": True,
"corporateHoster": checkHoster(result),
"hoster": result['isp'],
"countryCode": result['countryCode'],
"corporateStun": checkActiveOptions(page,bad_stuns),
"jitsiVersion": getJitsiVersion(page),
"authEnabled": checkActiveOptions(page,authOptions),
"analyticsEnabled": checkActiveOptions(page,analyticsOptions),
}
else:
output = {
"domain": domain,
"available": False,
"countryCode": "",
}
if result['status'] == "success":
output['countryCode'] = result['countryCode']

if available:
outputs.append(output)


Loading…
Cancel
Save