from urllib.parse import unquote, quote
from ioweb import request, DataNotValid
def main(**kwargs):
username = 'cal_89'
cursor_id = None
while True:
url = 'https://mobile.twitter.com/%s/followers' % (
quote(username),
)
if cursor_id:
url += '?cursor=%d' % cursor_id
res = request(url)
if not res.css('div.user-list').exists():
raise DataNotValid
usernames = set()
for elem in res.xpath(
'//a[strong[contains(@class, "fullname")]]'
):
usernames.add(
unquote(elem.attr('href').lstrip('/').split('?')[0])
.lower()
)
print(usernames)
# <div class="w-button-more"><a href="/dzuev/followers?cursor=1623574621321415001">
more = res.css('div.w-button-more a')
if more.exists():
cursor_id = int(
more.attr('href').split('cursor=')[1].split('&')[0]
)
else:
cursor_id = None
print('Cursor ID: %s' % cursor_id)
if not cursor_id:
print('Last page!')
break